diff --git "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" --- "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" +++ "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" @@ -63227,3 +63227,6214 @@ 2023-11-20 19:15:35,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log 2023-11-20 19:15:35,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive 2023-11-20 19:15:38,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:15:42,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:15:43,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:15:47,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:15:47,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:15:49,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:15:53,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:15:54,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:15:54,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:15:55,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:15:57,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:15:59,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:00,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:04,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:05,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:07,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:16:09,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:10,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:14,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:15,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:19,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:16:19,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:21,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:23,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:16:25,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:25,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:26,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:29,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:16:30,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:31,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:36,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:36,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:36,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:38,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:16:41,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:41,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:46,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:46,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:50,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:16:51,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:52,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:53,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:16:57,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:16:58,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:16:58,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:17:02,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:03,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:08,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:08,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:17:13,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:13,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:18,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:19,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:17:19,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:23,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:17:24,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:26,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:29,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:17:29,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:31,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:34,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:37,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:37,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:39,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:17:40,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:42,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:45,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:47,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:50,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:51,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:17:52,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:53,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:17:56,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:17:58,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:17:58,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:01,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:01,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:03,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:06,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:08,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:11,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:13,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:13,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:16,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:20,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:21,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:23,125 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:18:23,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:25,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:27,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:31,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:32,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:32,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:18:32,408 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:18:32,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:18:32,411 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:18:32,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:18:33,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:35,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:36,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:37,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:41,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:42,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:45,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:46,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:48,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:52,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:52,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:53,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:18:53,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:18:55,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:18:57,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:18:58,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:02,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:03,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:07,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:08,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:12,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:19:13,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:14,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:18,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:19,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:22,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:19:23,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:19:23,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:24,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:29,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:29,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:34,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:35,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:19:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:39,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:45,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:47,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:19:50,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:50,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:53,129 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:19:55,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:19:56,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:19:57,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:20:00,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:01,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:06,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:06,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:07,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:20:11,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:11,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:16,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:16,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:21,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:23,130 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:20:26,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:27,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:32,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:32,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:35,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:20:37,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:37,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:42,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:42,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:45,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:20:47,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:48,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:52,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:52,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:53,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:20:53,131 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:20:57,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:20:57,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:20:58,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:03,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:03,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:07,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:21:08,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:08,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:13,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:13,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:15,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:21:19,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:19,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:23,132 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:21:25,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:25,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:25,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:21:30,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:30,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:35,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:21:35,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:36,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:40,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:41,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:46,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:47,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:47,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:21:51,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:52,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:21:53,134 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:21:57,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:21:57,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:21:57,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:02,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:03,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:07,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:08,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:13,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:14,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:17,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:19,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:19,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:23,135 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:22:24,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:22:24,284 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:22:24,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:22:24,288 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:22:24,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:22:25,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:25,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:25,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:25,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:26,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:30,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:35,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:35,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:36,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:40,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:41,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:45,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:46,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:46,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:51,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:51,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:22:53,136 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:22:55,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:22:57,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:22:57,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:02,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:02,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:05,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:23:07,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:07,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:12,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:13,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:15,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:23:18,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:18,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:23,138 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:23:24,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:24,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:24,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:25,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:23:29,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:29,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:34,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:35,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:35,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:23:39,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:40,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:44,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:45,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:48,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:23:50,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:50,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:53,139 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:23:55,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:23:56,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:23:57,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:24:00,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:01,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:05,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:06,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:06,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:09,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:24:11,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:11,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:16,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:16,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:19,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:24:21,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:21,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:23,139 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:24:27,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:27,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:29,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:24:32,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:32,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:37,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:38,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:41,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:24:43,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:43,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:48,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:48,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:51,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:24:53,140 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:24:53,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:54,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:59,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:24:59,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:24:59,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:03,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:25:04,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:04,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:09,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:11,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:13,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:25:15,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:16,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:20,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:21,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:23,140 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:25:23,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:25:26,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:26,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:31,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:31,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:36,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:25:36,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:37,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:42,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:42,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:46,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:25:47,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:47,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:52,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:25:52,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:53,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:25:54,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:25:54,266 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:25:54,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:25:54,269 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:25:55,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:25:56,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:25:57,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:25:58,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:02,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:03,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:08,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:26:08,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:09,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:14,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:15,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:15,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:17,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:26:19,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:20,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:23,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:26:25,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:26,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:29,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:26:30,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:31,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:35,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:37,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:39,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:26:41,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:42,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:46,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:48,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:49,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:26:51,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:53,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:26:53,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:57,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:26:58,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:26:59,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:27:02,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:03,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:07,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:08,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:11,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:27:13,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:13,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:18,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:19,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:22,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:27:23,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:23,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:27:24,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:28,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:30,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:32,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:27:33,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:35,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:38,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:40,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:42,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:27:44,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:45,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:49,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:50,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:52,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:27:53,145 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:27:54,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:27:55,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:27:59,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:01,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:02,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:28:05,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:06,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:10,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:11,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:11,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:14,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:28:15,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:16,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:20,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:22,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:23,146 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:28:26,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:28:26,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:27,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:31,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:32,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:35,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:28:37,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:37,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:42,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:44,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:47,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:47,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:28:49,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:52,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:53,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:28:54,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:28:57,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:28:58,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:28:59,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:03,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:05,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:07,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:29:08,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:13,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:15,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:17,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:29:18,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:20,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:23,148 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:29:24,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:26,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:26,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:27,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:29:27,508 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:29:27,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:29:27,512 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:29:28,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:29:28,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:29:29,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:30,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:29:31,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:34,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:37,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:39,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:40,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:29:42,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:44,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:48,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:50,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:52,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:29:53,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:29:53,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:56,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:29:59,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:29:59,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:01,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:02,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:30:04,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:06,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:09,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:11,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:12,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:30:14,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:16,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:19,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:21,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:22,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:30:23,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:30:24,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:27,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:30,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:32,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:34,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:30:35,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:37,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:41,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:42,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:44,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:30:46,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:47,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:51,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:53,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:30:53,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:30:54,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:30:56,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:30:58,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:02,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:03,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:31:03,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:07,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:08,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:13,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:13,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:14,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:15,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:31:18,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:19,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:23,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:31:24,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:25,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:26,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:31:29,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:30,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:34,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:35,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:38,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:31:39,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:40,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:44,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:44,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:48,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:31:49,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:50,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:53,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:31:54,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:31:56,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:31:58,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:31:59,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:01,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:06,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:10,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:32:11,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:11,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:16,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:16,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:20,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:32:21,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:21,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:23,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:32:27,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:27,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:30,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:32:32,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:32,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:37,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:38,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:40,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:32:43,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:43,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:48,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:49,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:50,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:32:53,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:32:53,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:54,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:32:58,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:32:58,250 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:32:58,250 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:32:58,252 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:32:58,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:32:58,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:32:59,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:00,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:33:03,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:04,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:09,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:10,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:10,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:33:14,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:15,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:19,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:33:20,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:20,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:23,154 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:33:25,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:26,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:31,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:31,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:32,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:33:36,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:36,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:41,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:42,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:33:42,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:46,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:47,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:51,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:52,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:33:53,155 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:33:54,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:33:56,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:33:58,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:02,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:03,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:04,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:34:07,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:08,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:12,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:12,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:13,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:16,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:34:17,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:19,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:23,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:34:23,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:24,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:26,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:34:28,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:29,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:33,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:34,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:36,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:34:38,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:40,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:43,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:45,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:50,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:50,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:34:52,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:34:53,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:34:55,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:34:56,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:00,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:01,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:04,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:35:06,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:06,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:11,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:11,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:16,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:17,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:35:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:22,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:23,158 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:35:23,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:26,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:35:27,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:28,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:32,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:33,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:36,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:35:38,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:38,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:43,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:44,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:44,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:46,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:35:48,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:49,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:53,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:35:54,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:54,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:54,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:35:58,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:35:59,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:35:59,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:04,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:06,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:08,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:36:09,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:11,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:14,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:16,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:16,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:20,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:20,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:36:21,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:23,159 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:36:26,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:27,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:30,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:36:31,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:32,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:36,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:38,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:40,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:36:40,125 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:36:40,126 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:36:40,130 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:36:40,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:36:40,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:36:42,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:42,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:36:43,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:47,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:49,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:49,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:52,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:36:52,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:36:53,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:36:54,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:36:58,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:00,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:00,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:02,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:37:03,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:05,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:08,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:10,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:10,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:12,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:37:14,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:15,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:19,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:20,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:23,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:37:23,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:37:25,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:25,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:30,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:31,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:31,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:35,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:37:35,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:36,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:40,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:41,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:41,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:45,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:37:45,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:46,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:50,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:52,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:37:53,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:37:54,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:37:56,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:37:57,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:01,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:02,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:02,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:04,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:38:06,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:11,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:14,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:16,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:38:16,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:19,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:21,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:23,163 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:38:25,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:27,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:28,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:38:30,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:32,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:35,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:35,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:38,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:38,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:38:40,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:43,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:46,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:48,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:48,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:38:51,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:53,164 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:38:54,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:38:56,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:38:59,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:39:00,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:01,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:05,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:08,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:10,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:11,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:39:13,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:15,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:18,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:20,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:21,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:39:23,164 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:39:23,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:26,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:29,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:31,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:33,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:39:34,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:36,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:40,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:42,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:39:42,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:45,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:47,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:50,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:52,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:39:53,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:39:53,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:39:55,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:39:58,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:01,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:03,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:04,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:40:06,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:08,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:11,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:12,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:40:12,848 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:40:12,849 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:40:12,851 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:40:13,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:40:13,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:14,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:40:16,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:18,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:21,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:21,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:23,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:40:24,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:24,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:40:26,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:29,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:32,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:34,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:34,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:40:37,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:39,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:42,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:44,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:45,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:40:47,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:49,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:52,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:52,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:40:53,166 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:40:55,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:40:55,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:40:57,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:00,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:02,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:05,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:41:05,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:07,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:10,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:13,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:16,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:17,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:41:18,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:21,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:23,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:41:23,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:23,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:27,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:27,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:41:28,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:32,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:33,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:37,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:41:37,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:38,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:42,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:44,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:47,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:41:47,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:49,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:52,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:41:53,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:41:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:41:57,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:41:58,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:00,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:03,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:05,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:08,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:10,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:13,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:15,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:18,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:23,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:42:24,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:25,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:29,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:30,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:32,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:42:34,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:35,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:39,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:41,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:44,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:45,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:42:46,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:49,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:52,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:52,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:42:53,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:42:55,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:42:55,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:42:57,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:00,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:02,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:05,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:43:05,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:07,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:10,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:13,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:43:13,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:16,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:18,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:21,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:23,171 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:43:24,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:24,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:25,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:43:27,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:29,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:32,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:34,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:35,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:43:37,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:39,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:42,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:44,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:45,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:43:47,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:49,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:52,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:43:53,171 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:43:55,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:43:55,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:43:58,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:01,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:02,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:44:02,744 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:44:02,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:44:02,747 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:44:03,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:44:03,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:05,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:44:06,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:08,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:12,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:13,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:44:14,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:17,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:19,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:23,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:23,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:23,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:44:24,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:25,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:44:28,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:29,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:33,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:34,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:44:35,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:38,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:40,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:44,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:44,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:44:45,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:49,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:50,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:44:53,174 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:44:55,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:44:55,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:44:56,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:00,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:05,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:45:05,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:06,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:10,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:11,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:15,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:45:16,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:17,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:22,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:22,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:22,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:23,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:45:25,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:45:27,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:27,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:32,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:33,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:35,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:45:38,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:38,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:43,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:44,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:45,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:45:49,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:49,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:53,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:45:54,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:54,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:54,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:45:55,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:45:59,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:45:59,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:04,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:05,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:05,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:46:10,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:10,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:15,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:46:15,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:16,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:21,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:22,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:22,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:23,177 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:46:25,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:46:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:27,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:31,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:32,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:35,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:46:37,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:37,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:42,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:43,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:45,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:46:47,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:48,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:52,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:53,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:53,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:46:53,177 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:46:55,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:46:57,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:46:58,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:02,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:03,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:07,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:07,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:08,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:12,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:13,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:13,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:17,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:18,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:18,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:23,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:23,179 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:47:24,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:25,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:47:25,192 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:47:25,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:47:25,196 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:47:26,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:47:27,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:28,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:28,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:29,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:33,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:34,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:37,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:38,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:39,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:43,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:44,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:47,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:48,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:50,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:53,181 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:47:54,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:47:55,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:47:57,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:47:59,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:00,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:04,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:05,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:07,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:48:10,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:15,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:16,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:17,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:48:20,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:21,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:23,181 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:48:26,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:27,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:27,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:48:31,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:33,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:37,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:37,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:48:38,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:42,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:44,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:47,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:47,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:48:49,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:52,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:48:53,183 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:48:55,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:48:55,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:48:57,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:00,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:02,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:05,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:05,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:49:08,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:11,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:13,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:15,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:49:16,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:19,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:23,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:23,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:23,196 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:49:24,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:26,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:49:28,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:29,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:33,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:34,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:36,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:49:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:39,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:44,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:45,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:46,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:49:49,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:50,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:53,184 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:49:54,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:49:56,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:49:59,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:01,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:04,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:06,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:07,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:50:09,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:11,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:14,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:17,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:50:17,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:20,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:22,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:23,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:50:26,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:27,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:50:28,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:31,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:33,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:36,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:38,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:41,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:43,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:43,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:50:46,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:48,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:51,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:53,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:50:53,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:50:54,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:50:56,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:50:59,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:00,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:51:00,886 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:51:00,887 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:51:00,890 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:51:01,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:51:01,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:51:02,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:04,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:07,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:09,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:11,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:51:13,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:15,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:19,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:21,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:21,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:51:23,187 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:51:24,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:27,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:30,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:30,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:33,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:33,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:51:35,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:38,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:40,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:43,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:44,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:51:45,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:48,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:50,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:50,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:53,187 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:51:54,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:51:54,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:51:55,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:51:59,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:02,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:04,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:06,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:52:07,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:09,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:13,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:15,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:16,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:52:18,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:20,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:23,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:52:23,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:26,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:27,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:52:28,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:31,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:33,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:36,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:38,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:39,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:52:41,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:43,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:46,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:48,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:49,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:52:52,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:53,190 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:52:53,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:53,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:58,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:52:59,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:52:59,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:53:03,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:04,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:08,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:09,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:53:10,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:10,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:14,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:15,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:19,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:20,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:21,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:53:23,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:53:25,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:25,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:30,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:31,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:33,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:53:35,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:36,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:40,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:41,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:44,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:53:45,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:46,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:50,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:52,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:52,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:53:53,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:53:54,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:53:56,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:53:57,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:01,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:02,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:04,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:54:06,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:07,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:11,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:12,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:14,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:54:16,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:17,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:21,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:23,193 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:54:23,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:24,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:54:27,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:28,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:32,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:33,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:54:33,746 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:54:33,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:54:33,751 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:54:34,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:34,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:54:36,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:54:37,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:39,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:42,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:44,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:45,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:54:48,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:49,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:53,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:54:53,196 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:54:54,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:54,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:54:55,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:54:58,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:00,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:03,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:05,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:55:05,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:08,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:10,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:13,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:15,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:55:16,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:19,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:21,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:23,196 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:55:24,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:25,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:55:26,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:26,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:30,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:31,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:35,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:37,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:37,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:55:40,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:42,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:45,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:47,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:48,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:55:51,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:53,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:55:53,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:55:57,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:55:58,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:02,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:04,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:08,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:08,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:56:09,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:13,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:15,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:18,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:56:18,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:20,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:23,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:56:24,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:26,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:26,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:28,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:56:29,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:31,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:34,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:37,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:40,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:40,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:56:42,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:45,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:47,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:48,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:56:50,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:52,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:53,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:56:56,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:56:58,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:58,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:56:58,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:01,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:03,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:07,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:08,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:09,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:12,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:14,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:14,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:17,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:18,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:19,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:22,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:23,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:57:25,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:26,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:28,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:30,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:33,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:35,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:37,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:38,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:40,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:43,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:46,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:47,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:49,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:51,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:57:53,200 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:57:55,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:57:55,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:57:57,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:00,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:03,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:03,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:04,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 19:58:04,613 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 19:58:04,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 19:58:04,616 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 19:58:05,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 19:58:05,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:06,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:58:08,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:10,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:13,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:13,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:16,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:16,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:58:18,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:21,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:23,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:58:24,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:26,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:58:26,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:29,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:31,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:35,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:36,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:58:37,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:40,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:42,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:45,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:46,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:58:48,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:50,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:53,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:58:54,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:58:56,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:58:56,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:58:59,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:02,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:04,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:06,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:59:07,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:09,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:13,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:13,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:15,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:16,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:59:18,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:20,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:23,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:59:23,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:25,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:26,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:59:28,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:30,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:34,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:36,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:36,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:59:39,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:45,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:45,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:46,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:59:47,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:51,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:52,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 19:59:53,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 19:59:54,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 19:59:56,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 19:59:58,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:02,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:03,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:04,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:00:07,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:08,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:12,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:12,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:13,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:14,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:00:17,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:18,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:22,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:23,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:00:23,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:00:24,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:27,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:33,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:34,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:00:34,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:38,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:39,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:44,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:00:45,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:48,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:50,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:53,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:00:54,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:00:54,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:00:55,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:00:59,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:00,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:04,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:04,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:05,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:09,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:10,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:14,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:15,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:15,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:16,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:20,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:21,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:23,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:01:23,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:01:23,938 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:01:23,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:01:23,942 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:01:24,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:01:24,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:25,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:26,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:26,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:30,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:31,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:36,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:36,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:37,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:42,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:42,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:44,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:47,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:48,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:52,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:53,207 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:01:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:01:54,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:01:57,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:01:59,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:03,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:04,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:02:05,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:08,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:10,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:14,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:15,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:16,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:02:19,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:20,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:23,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:02:24,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:02:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:26,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:30,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:31,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:35,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:02:36,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:37,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:41,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:42,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:45,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:02:46,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:47,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:51,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:53,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:02:53,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:02:54,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:02:56,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:02:58,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:02,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:02,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:03,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:04,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:03:07,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:08,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:12,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:14,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:14,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:03:18,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:19,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:23,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:03:23,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:24,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:03:25,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:28,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:30,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:33,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:34,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:03:36,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:38,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:41,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:44,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:44,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:03:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:49,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:51,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:03:53,210 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:03:54,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:03:55,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:03:57,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:01,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:02,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:04,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:04:06,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:07,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:11,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:12,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:14,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:04:16,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:18,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:22,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:23,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:04:23,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:25,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:04:27,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:33,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:33,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:35,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:04:38,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:38,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:42,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:04:42,354 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:04:42,355 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:04:42,357 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:04:43,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:04:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:44,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:45,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:04:49,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:49,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:53,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:04:54,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:04:55,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:04:55,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:00,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:00,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:00,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:03,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:05,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:05,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:10,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:11,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:12,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:16,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:16,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:21,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:22,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:23,212 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:05:24,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:27,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:32,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:32,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:34,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:37,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:38,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:43,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:43,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:44,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:48,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:48,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:53,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:53,223 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:05:53,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:05:58,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:05:58,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:05:59,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:03,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:04,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:08,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:06:08,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:09,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:13,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:14,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:18,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:06:19,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:20,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:23,213 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:06:25,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:26,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:28,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:06:30,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:31,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:35,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:37,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:38,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:06:41,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:42,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:46,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:47,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:49,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:06:51,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:52,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:53,213 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:06:57,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:06:58,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:06:59,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:07:02,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:03,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:07,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:08,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:09,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:07:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:15,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:15,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:18,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:19,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:07:20,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:23,215 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:07:24,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:25,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:27,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:07:29,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:30,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:34,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:36,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:37,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:07:39,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:41,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:44,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:47,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:49,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:07:50,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:52,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:53,216 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:07:56,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:07:57,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:07:59,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:01,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:03,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:05,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:08:05,992 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:08:05,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:08:05,995 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:08:06,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:08:06,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:06,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:08,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:11,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:14,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:14,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:16,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:17,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:19,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:22,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:23,217 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:08:25,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:26,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:28,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:30,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:33,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:35,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:36,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:39,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:40,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:44,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:46,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:46,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:49,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:51,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:53,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:08:54,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:08:56,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:08:56,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:08:59,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:01,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:05,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:07,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:09:07,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:10,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:12,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:15,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:17,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:09:17,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:20,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:23,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:09:23,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:25,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:27,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:09:28,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:30,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:33,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:33,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:35,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:37,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:09:38,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:40,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:44,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:45,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:09:46,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:49,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:51,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:53,224 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:09:54,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:09:56,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:09:57,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:09:59,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:01,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:04,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:06,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:07,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:10:10,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:11,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:16,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:17,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:17,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:10:21,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:22,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:23,224 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:10:25,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:10:26,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:28,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:32,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:33,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:35,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:10:37,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:38,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:43,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:44,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:45,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:10:48,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:49,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:53,225 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:10:54,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:10:54,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:10:55,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:10:59,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:00,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:04,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:04,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:11:06,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:09,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:11,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:14,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:11:15,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:16,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:21,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:21,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:23,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:11:23,044 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:11:23,044 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:11:23,046 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:11:23,225 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:11:24,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:11:25,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:11:26,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:27,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:32,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:32,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:32,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:35,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:11:37,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:37,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:42,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:43,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:43,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:11:47,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:48,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:53,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:11:53,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:53,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:11:55,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:11:58,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:11:58,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:03,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:04,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:05,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:12:08,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:11,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:13,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:12:14,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:16,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:19,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:21,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:23,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:12:23,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:12:24,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:26,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:29,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:32,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:32,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:33,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:12:34,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:37,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:39,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:42,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:43,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:12:45,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:47,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:50,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:53,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:12:53,227 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:12:53,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:12:56,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:12:58,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:01,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:03,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:03,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:03,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:13:06,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:08,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:12,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:13,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:13:14,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:17,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:20,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:20,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:23,227 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:13:23,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:23,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:13:25,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:28,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:30,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:33,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:13:33,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:36,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:38,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:41,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:43,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:13:43,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:46,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:48,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:52,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:53,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:13:53,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:13:54,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:13:57,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:13:59,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:02,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:03,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:14:04,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:08,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:09,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:13,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:14:13,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:15,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:19,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:19,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:20,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:23,229 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:14:23,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:14:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:25,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:30,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:30,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:31,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:14:35,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:35,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:40,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:40,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:40,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:14:40,735 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:14:40,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:14:40,738 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:14:41,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:14:43,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:14:45,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:45,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:50,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:50,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:14:53,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:14:53,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:14:55,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:14:56,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:00,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:01,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:03,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:15:06,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:06,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:11,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:12,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:13,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:15:16,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:17,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:21,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:23,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:23,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:15:23,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:15:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:28,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:32,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:33,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:15:37,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:38,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:42,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:44,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:45,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:15:47,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:49,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:53,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:15:53,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:15:53,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:15:55,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:15:58,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:00,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:04,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:04,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:04,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:16:06,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:09,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:11,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:13,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:16:14,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:17,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:20,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:20,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:22,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:23,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:16:23,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:16:25,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:28,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:31,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:33,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:35,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:16:36,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:38,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:41,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:44,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:45,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:16:46,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:49,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:52,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:16:53,231 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:16:53,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:16:55,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:16:57,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:00,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:02,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:03,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:17:06,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:07,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:11,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:13,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:17:13,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:16,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:18,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:22,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:23,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:17:23,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:17:24,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:28,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:30,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:33,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:17:34,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:35,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:39,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:40,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:43,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:17:44,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:45,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:49,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:50,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:50,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:17:53,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:17:53,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:17:55,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:17:55,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:00,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:00,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:01,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:18:01,092 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:18:01,093 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:18:01,095 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:18:01,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:18:03,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:18:05,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:06,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:11,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:11,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:11,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:18:16,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:16,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:21,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:22,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:18:22,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:23,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:18:27,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:28,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:32,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:33,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:34,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:18:38,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:38,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:43,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:44,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:18:44,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:48,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:49,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:18:53,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:18:54,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:18:54,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:18:55,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:00,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:00,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:03,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:19:05,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:05,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:10,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:11,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:13,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:19:16,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:16,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:21,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:21,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:23,234 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:19:23,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:19:26,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:27,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:32,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:32,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:33,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:19:37,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:37,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:42,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:43,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:43,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:19:47,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:48,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:53,234 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:19:53,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:19:53,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:19:54,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:19:58,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:00,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:03,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:03,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:20:05,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:08,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:10,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:13,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:13,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:20:15,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:18,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:20,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:23,235 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:20:23,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:20:24,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:29,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:30,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:34,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:35,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:35,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:38,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:20:40,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:40,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:45,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:46,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:48,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:20:50,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:51,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:53,236 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:20:56,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:20:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:20:58,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:21:01,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:02,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:07,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:07,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:08,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:21:12,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:12,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:18,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:18,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:21:18,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:23,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:23,239 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:21:24,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:25,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:21:25,594 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:21:25,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:21:25,597 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:21:26,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:21:28,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:21:28,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:29,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:33,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:35,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:35,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:38,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:21:39,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:40,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:44,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:46,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:48,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:21:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:51,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:53,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:21:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:21:56,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:21:57,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:01,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:06,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:07,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:07,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:11,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:12,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:17,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:17,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:17,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:22,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:23,241 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:22:24,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:27,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:28,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:29,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:33,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:34,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:34,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:35,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:38,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:39,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:43,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:45,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:47,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:49,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:50,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:53,243 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:22:54,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:22:55,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:22:58,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:22:59,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:00,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:04,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:06,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:08,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:23:10,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:11,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:16,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:23:16,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:16,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:21,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:22,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:23,243 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:23:26,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:23:27,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:27,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:32,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:34,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:37,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:38,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:23:39,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:42,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:45,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:46,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:23:48,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:50,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:53,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:23:53,246 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:23:55,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:23:56,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:23:58,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:00,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:06,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:06,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:06,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:24:09,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:11,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:14,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:16,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:16,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:24:19,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:21,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:23,247 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:24:25,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:26,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:26,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:24:30,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:31,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:36,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:36,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:24:37,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:41,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:42,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:45,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:24:45,120 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:24:45,121 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:24:45,123 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:24:45,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:24:45,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:24:47,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:47,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:53,249 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:24:53,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:24:55,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:24:57,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:24:58,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:02,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:04,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:05,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:25:08,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:09,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:13,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:14,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:16,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:25:19,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:19,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:23,250 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:25:24,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:25,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:26,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:25:29,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:30,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:35,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:35,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:36,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:25:40,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:40,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:46,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:46,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:25:46,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:51,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:51,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:25:53,251 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:25:56,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:25:56,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:25:57,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:01,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:03,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:06,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:26:07,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:08,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:12,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:13,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:16,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:26:17,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:22,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:23,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:26:24,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:26,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:26:28,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:29,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:33,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:35,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:36,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:26:39,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:40,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:44,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:45,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:46,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:26:49,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:52,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:52,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:52,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:26:53,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:26:54,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:26:55,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:26:57,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:00,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:02,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:04,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:27:06,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:07,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:11,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:13,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:14,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:27:16,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:18,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:23,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:27:24,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:25,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:27:27,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:29,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:32,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:34,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:27:34,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:37,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:39,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:42,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:44,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:27:44,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:47,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:49,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:52,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:27:53,254 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:27:54,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:27:55,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:27:58,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:01,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:02,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:28:02,925 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:28:02,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:28:02,928 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:28:03,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:28:03,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:04,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:28:06,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:08,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:12,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:14,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:16,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:28:17,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:19,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:22,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:22,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:22,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:23,255 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:28:24,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:28:25,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:27,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:30,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:33,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:34,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:28:36,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:38,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:41,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:44,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:46,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:28:47,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:49,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:52,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:53,256 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:28:54,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:28:56,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:28:58,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:28:59,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:03,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:05,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:06,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:29:09,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:10,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:14,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:16,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:16,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:29:19,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:22,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:23,257 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:29:24,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:29:25,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:27,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:30,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:33,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:34,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:29:36,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:38,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:41,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:43,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:45,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:29:46,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:48,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:51,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:53,268 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:29:54,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:29:55,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:29:57,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:29:59,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:02,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:04,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:05,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:30:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:09,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:12,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:14,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:30:14,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:18,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:19,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:23,258 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:30:23,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:24,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:30:25,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:28,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:30,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:34,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:30:34,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:35,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:39,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:40,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:45,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:45,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:46,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:30:50,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:50,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:53,259 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:30:55,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:30:56,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:30:56,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:00,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:01,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:05,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:06,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:06,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:12,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:14,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:15,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:17,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:20,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:23,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:23,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:23,259 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:31:23,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:31:23,322 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:31:23,323 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:31:23,326 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:31:23,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:31:24,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:26,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:28,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:31,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:34,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:34,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:37,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:39,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:42,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:44,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:44,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:48,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:53,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:31:53,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:31:55,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:31:57,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:31:58,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:00,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:03,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:05,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:07,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:32:09,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:10,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:15,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:16,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:32:20,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:21,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:23,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:32:25,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:26,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:27,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:32:30,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:31,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:35,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:36,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:37,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:32:40,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:43,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:46,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:32:46,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:48,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:51,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:53,264 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:32:53,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:53,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:32:56,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:32:57,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:32:58,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:02,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:03,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:06,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:33:07,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:08,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:12,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:13,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:16,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:33:18,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:18,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:23,265 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:33:24,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:24,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:24,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:26,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:33:29,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:29,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:34,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:34,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:34,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:36,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:33:39,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:40,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:45,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:46,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:46,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:33:50,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:51,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:53,267 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:33:55,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:33:56,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:33:57,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:00,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:01,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:05,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:07,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:07,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:11,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:12,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:16,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:17,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:18,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:21,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:23,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:23,268 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:34:27,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:27,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:28,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:32,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:33,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:37,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:37,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:39,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:42,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:44,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:46,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:34:46,383 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:34:46,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:34:46,491 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:34:47,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:34:48,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:48,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:49,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:50,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:53,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:34:53,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:34:56,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:34:57,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:34:58,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:01,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:05,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:05,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:06,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:09,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:35:10,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:11,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:15,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:16,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:20,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:21,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:23,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:35:23,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:35:25,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:27,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:31,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:32,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:33,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:35:36,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:37,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:40,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:35:42,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:43,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:47,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:48,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:50,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:35:52,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:53,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:35:54,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:35:58,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:35:59,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:00,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:36:03,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:04,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:09,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:09,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:09,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:10,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:36:14,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:15,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:19,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:20,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:21,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:36:23,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:36:24,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:26,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:30,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:31,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:36:31,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:35,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:36,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:40,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:41,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:36:41,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:45,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:46,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:51,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:51,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:36:52,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:36:53,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:36:56,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:36:58,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:01,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:37:01,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:09,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:11,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:37:13,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:14,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:20,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:21,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:37:23,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:37:23,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:25,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:29,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:29,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:37:30,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:34,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:35,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:39,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:37:39,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:41,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:45,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:46,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:49,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:37:50,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:52,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:53,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:37:56,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:37:57,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:37:59,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:01,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:02,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:07,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:07,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:07,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:08,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:38:08,590 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:38:08,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:38:08,594 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:38:08,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:38:08,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:10,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:12,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:13,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:17,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:18,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:19,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:22,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:23,271 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:38:25,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:28,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:29,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:31,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:33,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:36,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:38,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:39,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:41,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:43,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:46,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:48,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:49,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:38:52,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:53,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:38:53,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:38:57,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:38:59,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:01,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:39:02,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:04,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:09,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:39:09,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:13,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:16,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:19,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:19,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:39:21,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:23,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:39:24,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:27,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:29,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:39:29,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:32,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:34,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:38,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:38,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:39,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:39:40,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:43,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:45,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:49,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:49,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:39:50,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:53,274 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:39:54,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:39:56,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:39:59,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:40:00,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:01,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:05,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:06,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:09,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:40:10,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:12,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:16,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:17,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:19,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:40:21,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:22,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:23,274 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:40:27,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:28,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:31,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:40:32,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:33,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:38,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:39,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:40:39,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:43,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:44,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:49,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:40:49,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:53,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:40:54,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:40:56,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:40:59,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:41:00,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:01,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:05,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:06,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:10,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:10,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:11,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:41:11,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:16,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:17,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:19,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:41:22,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:23,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:41:27,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:28,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:28,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:41:28,106 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:41:28,107 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:41:28,110 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:41:28,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:41:29,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:41:32,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:33,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:38,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:38,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:39,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:41:43,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:43,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:48,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:49,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:49,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:41:53,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:41:53,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:54,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:41:59,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:41:59,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:42:00,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:05,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:05,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:07,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:42:10,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:10,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:15,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:16,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:17,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:42:21,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:21,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:23,277 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:42:26,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:26,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:27,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:42:31,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:31,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:36,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:36,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:38,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:42:41,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:41,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:46,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:46,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:48,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:42:51,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:51,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:53,278 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:42:57,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:57,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:42:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:42:58,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:43:02,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:02,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:07,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:43:07,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:07,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:12,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:12,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:17,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:43:18,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:18,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:23,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:23,280 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:43:24,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:27,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:43:29,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:29,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:34,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:34,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:37,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:43:39,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:39,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:44,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:45,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:47,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:43:50,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:50,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:53,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:43:55,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:43:55,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:55,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:43:57,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:00,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:00,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:05,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:06,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:06,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:11,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:11,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:15,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:16,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:16,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:21,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:21,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:23,282 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:44:25,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:27,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:27,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:32,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:32,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:35,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:37,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:38,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:43,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:44,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:44:45,342 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:44:45,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:44:45,345 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:44:45,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:44:47,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:48,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:49,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:53,283 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:44:54,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:44:54,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:56,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:44:59,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:44:59,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:05,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:05,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:06,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:45:10,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:10,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:16,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:45:16,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:16,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:21,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:21,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:23,285 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:45:26,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:45:26,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:27,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:32,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:32,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:36,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:45:37,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:37,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:42,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:43,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:43,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:46,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:45:48,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:48,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:53,298 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:45:53,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:54,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:45:56,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:45:59,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:45:59,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:04,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:05,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:05,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:46:09,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:10,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:14,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:15,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:46:16,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:19,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:21,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:23,286 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:46:25,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:25,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:46:26,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:30,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:31,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:35,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:46:36,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:37,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:42,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:42,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:42,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:45,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:46:47,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:47,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:52,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:46:53,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:46:53,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:55,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:46:58,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:46:58,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:03,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:47:04,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:04,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:09,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:09,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:13,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:47:14,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:15,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:19,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:20,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:23,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:47:24,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:47:25,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:25,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:30,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:31,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:34,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:47:36,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:36,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:41,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:42,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:42,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:44,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:47:46,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:47,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:51,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:53,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:47:53,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:47:54,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:47:57,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:47:58,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:01,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:48:01,596 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:48:01,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:48:01,600 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:48:02,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:48:02,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:03,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:03,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:04,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:07,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:09,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:09,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:12,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:13,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:14,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:18,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:20,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:22,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:23,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:23,288 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:48:25,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:30,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:32,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:33,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:35,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:38,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:40,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:41,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:42,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:44,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:46,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:49,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:51,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:51,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:48:53,289 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:48:54,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:48:56,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:48:59,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:01,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:49:01,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:05,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:06,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:10,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:11,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:49:12,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:15,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:17,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:20,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:21,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:49:23,289 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:49:23,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:25,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:29,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:29,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:29,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:30,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:32,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:49:34,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:35,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:40,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:41,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:42,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:49:45,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:46,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:50,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:52,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:49:52,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:49:53,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:49:55,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:49:57,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:01,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:02,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:06,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:50:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:07,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:11,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:12,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:16,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:17,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:21,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:22,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:23,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:50:26,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:31,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:33,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:36,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:38,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:41,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:43,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:46,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:48,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:51,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:53,291 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:50:54,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:50:56,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:50:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:01,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:04,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:06,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:12,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:14,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:51:14,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:17,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:19,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:22,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:23,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:51:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:26,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:51:27,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:30,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:32,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:35,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:36,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:51:37,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:40,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:42,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:43,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:51:46,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:47,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:51,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:53,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:51:53,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:51:53,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:51:57,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:51:58,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:02,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:04,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:04,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:52:08,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:09,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:13,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:14,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:16,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:52:19,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:19,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:23,295 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:52:23,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:52:23,888 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:52:23,889 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:52:23,894 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:52:24,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:52:24,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:52:24,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:25,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:26,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:52:29,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:30,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:34,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:35,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:39,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:40,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:44,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:52:45,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:45,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:50,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:52:51,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:53,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:52:54,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:52:56,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:52:56,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:01,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:04,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:53:06,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:07,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:12,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:12,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:16,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:53:17,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:17,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:22,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:22,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:23,295 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:53:26,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:53:27,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:27,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:32,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:32,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:36,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:53:37,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:38,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:43,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:43,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:43,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:46,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:53:48,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:49,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:53,297 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:53:54,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:54,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:53:56,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:53:59,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:53:59,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:04,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:04,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:09,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:54:09,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:10,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:14,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:14,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:14,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:15,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:18,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:54:19,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:23,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:54:24,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:26,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:26,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:54:29,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:31,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:35,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:36,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:38,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:54:40,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:41,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:46,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:46,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:54:47,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:51,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:52,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:54:53,299 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:54:56,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:54:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:54:57,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:01,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:02,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:06,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:07,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:08,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:12,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:13,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:16,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:17,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:19,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:22,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:23,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:55:25,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:26,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:27,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:30,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:33,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:35,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:36,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:38,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:40,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:43,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:45,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:46,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:48,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:50,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:53,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:55:53,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:55:53,309 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:55:53,309 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:55:53,311 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:55:53,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:55:53,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:54,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:55:54,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:55:56,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:55:59,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:01,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:04,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:04,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:56:07,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:09,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:12,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:15,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:56:15,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:18,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:21,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:23,301 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:56:23,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:25,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:56:26,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:28,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:32,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:33,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:37,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:56:37,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:38,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:43,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:44,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:45,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:56:48,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:49,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:53,302 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:56:53,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:56:54,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:56:55,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:56:58,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:00,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:04,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:57:04,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:05,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:09,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:10,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:14,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:57:15,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:16,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:21,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:21,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:23,302 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:57:24,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:57:26,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:27,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:32,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:32,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:32,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:34,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:57:37,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:37,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:42,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:42,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:44,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:57:47,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:48,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:52,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:53,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:57:53,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:57:54,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:57:57,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:57:58,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:03,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:03,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:04,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:58:08,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:08,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:14,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:14,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:14,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:58:19,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:19,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:22,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:58:23,304 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:58:25,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:25,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:30,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:30,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:30,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:30,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:33,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:58:35,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:36,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:41,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:41,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:45,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:58:46,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:46,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:51,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:58:52,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:53,304 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:58:55,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:58:57,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:58:57,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:02,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:03,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:03,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:05,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:07,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:08,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:12,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:13,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:14,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 20:59:14,355 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 20:59:14,356 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 20:59:14,359 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 20:59:14,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 20:59:16,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:17,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:18,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:18,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:23,305 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:59:24,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:24,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:26,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:29,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:30,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:34,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:34,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:35,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:36,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:39,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:41,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:45,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:46,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:46,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:50,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:51,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 20:59:53,306 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 20:59:55,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 20:59:56,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 20:59:56,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:00,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:01,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:06,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:06,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:00:07,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:11,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:12,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:17,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:17,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:18,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:00:22,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:22,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:23,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:00:27,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:28,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:28,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:00:32,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:38,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:39,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:00:39,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:43,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:44,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:48,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:49,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:00:49,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:53,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:00:53,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:55,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:00:58,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:00:59,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:01:00,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:03,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:05,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:09,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:01:09,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:11,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:15,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:16,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:19,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:01:20,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:22,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:23,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:01:26,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:28,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:29,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:01:31,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:33,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:37,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:38,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:39,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:01:42,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:43,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:48,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:49,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:01:49,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:53,308 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:01:53,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:55,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:01:58,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:01:59,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:00,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:03,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:05,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:08,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:09,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:10,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:13,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:15,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:18,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:18,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:21,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:23,308 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:02:23,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:26,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:28,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:29,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:31,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:35,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:36,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:02:36,265 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:02:36,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:02:36,268 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:02:36,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:02:36,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:37,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:38,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:40,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:42,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:45,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:46,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:47,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:50,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:52,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:02:53,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:02:56,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:02:57,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:02:58,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:01,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:03,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:08,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:09,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:03:11,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:13,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:17,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:17,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:19,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:03:19,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:22,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:23,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:03:25,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:27,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:29,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:03:31,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:32,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:36,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:38,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:39,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:03:41,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:43,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:47,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:03:47,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:49,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:52,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:53,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:03:54,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:03:58,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:03:59,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:01,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:04:04,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:04,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:09,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:10,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:11,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:04:14,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:15,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:19,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:21,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:04:21,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:23,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:04:24,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:28,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:30,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:31,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:04:33,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:38,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:40,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:41,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:04:43,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:45,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:49,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:49,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:51,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:51,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:04:53,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:04:54,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:04:57,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:04:59,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:00,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:05:03,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:04,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:08,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:10,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:11,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:05:13,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:15,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:18,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:19,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:05:21,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:23,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:05:23,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:27,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:28,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:29,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:05:32,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:33,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:38,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:41,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:05:43,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:44,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:49,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:49,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:50,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:51,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:05:53,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:05:54,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:05:56,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:05:59,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:01,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:04,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:06:04,209 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:06:04,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:06:04,211 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:06:04,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:06:04,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:05,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:06,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:07,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:09,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:12,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:14,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:15,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:17,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:19,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:22,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:23,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:06:25,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:25,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:27,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:30,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:32,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:37,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:40,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:42,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:45,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:47,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:48,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:50,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:53,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:06:54,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:06:56,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:06:57,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:06:59,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:01,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:04,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:06,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:07,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:07:10,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:13,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:15,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:07:15,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:18,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:20,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:23,315 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:07:24,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:24,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:25,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:07:26,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:29,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:31,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:34,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:34,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:37,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:37,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:07:39,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:42,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:44,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:47,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:47,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:07:49,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:52,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:07:53,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:07:55,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:07:55,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:07:57,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:00,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:02,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:05,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:08:05,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:08,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:11,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:13,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:15,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:08:16,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:19,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:22,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:22,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:23,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:08:24,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:25,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:08:27,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:29,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:32,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:34,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:35,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:08:37,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:37,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:39,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:43,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:45,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:45,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:08:48,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:50,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:53,317 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:08:54,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:08:55,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:08:55,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:08:59,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:00,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:04,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:05,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:06,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:11,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:11,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:13,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:16,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:16,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:22,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:22,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:22,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:09:22,632 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:09:22,633 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:09:22,635 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:09:23,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:09:23,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:09:24,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:25,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:27,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:27,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:32,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:33,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:36,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:37,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:38,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:42,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:43,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:44,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:48,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:53,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:09:53,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:54,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:09:54,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:54,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:09:58,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:09:59,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:03,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:04,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:10:04,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:08,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:11,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:13,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:10:13,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:16,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:18,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:21,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:23,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:10:23,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:10:24,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:26,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:29,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:32,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:33,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:10:34,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:37,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:39,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:43,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:43,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:10:45,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:48,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:50,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:10:53,326 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:10:53,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:10:54,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:54,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:10:56,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:00,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:01,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:03,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:11:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:06,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:11,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:12,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:13,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:11:16,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:17,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:22,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:22,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:23,327 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:11:23,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:23,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:11:27,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:28,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:32,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:33,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:33,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:11:37,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:43,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:43,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:11:44,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:48,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:49,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:53,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:11:53,327 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:11:54,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:11:55,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:11:58,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:00,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:04,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:04,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:05,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:10,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:10,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:10,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:14,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:15,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:15,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:20,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:21,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:23,329 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:12:24,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:26,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:26,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:31,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:32,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:34,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:36,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:37,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:41,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:12:41,503 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:12:41,503 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:12:41,506 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:12:42,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:42,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:12:42,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:43,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:44,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:47,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:51,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:12:52,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:53,329 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:12:53,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:12:57,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:12:58,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:01,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:13:03,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:03,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:08,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:10,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:10,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:11,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:13:14,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:15,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:19,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:20,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:21,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:13:23,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:13:25,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:25,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:30,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:31,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:31,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:13:35,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:36,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:41,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:41,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:13:42,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:46,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:47,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:51,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:13:52,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:53,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:13:53,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:13:58,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:13:59,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:02,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:14:03,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:04,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:08,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:10,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:10,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:12,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:14:14,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:15,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:19,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:21,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:22,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:14:23,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:14:25,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:26,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:31,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:32,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:32,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:14:36,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:37,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:41,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:42,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:14:42,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:46,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:47,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:51,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:52,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:14:53,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:14:54,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:14:57,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:14:57,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:02,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:15:03,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:03,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:08,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:09,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:09,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:12,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:15:13,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:14,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:18,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:20,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:22,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:15:23,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:15:24,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:25,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:29,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:31,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:33,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:15:34,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:36,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:39,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:41,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:43,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:15:45,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:46,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:50,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:52,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:15:53,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:15:53,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:15:55,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:15:57,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:00,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:01,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:16:01,428 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:16:01,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:16:01,431 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:16:01,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:16:03,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:03,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:16:05,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:08,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:10,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:13,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:16:14,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:16,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:20,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:21,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:23,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:16:24,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:16:25,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:27,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:30,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:32,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:34,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:16:35,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:37,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:41,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:43,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:44,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:16:46,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:48,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:52,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:53,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:16:54,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:16:54,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:16:57,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:16:59,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:03,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:04,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:17:05,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:08,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:10,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:13,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:14,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:17:16,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:18,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:21,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:23,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:17:23,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:24,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:17:26,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:28,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:31,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:34,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:17:34,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:36,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:39,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:41,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:44,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:17:45,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:47,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:51,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:53,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:17:53,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:17:54,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:17:56,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:17:58,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:01,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:03,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:04,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:18:06,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:08,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:12,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:12,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:18:14,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:17,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:19,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:21,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:18:22,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:23,346 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:18:24,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:27,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:29,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:33,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:34,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:18:34,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:38,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:39,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:42,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:18:43,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:45,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:49,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:50,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:18:52,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:18:53,335 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:18:54,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:18:55,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:00,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:00,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:02,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:05,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:05,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:10,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:11,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:12,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:15,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:16,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:20,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:19:20,129 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:19:20,130 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:19:20,132 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:19:20,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:19:21,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:21,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:21,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:22,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:23,336 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:19:26,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:26,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:31,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:32,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:32,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:36,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:37,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:42,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:42,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:42,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:47,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:47,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:52,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:52,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:19:53,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:19:53,336 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:19:57,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:19:59,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:02,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:20:03,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:04,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:08,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:10,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:10,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:12,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:20:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:15,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:18,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:21,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:22,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:20:23,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:20:24,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:26,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:29,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:31,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:34,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:36,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:36,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:38,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:20:39,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:41,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:44,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:47,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:48,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:20:50,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:52,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:53,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:20:56,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:20:58,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:20:59,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:01,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:03,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:06,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:08,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:09,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:11,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:14,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:16,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:19,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:20,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:21,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:23,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:21:24,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:27,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:30,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:30,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:32,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:36,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:38,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:38,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:41,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:46,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:48,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:49,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:51,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:53,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:21:54,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:21:57,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:21:58,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:21:59,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:02,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:04,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:08,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:08,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:10,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:13,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:15,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:18,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:18,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:20,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:23,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:22:25,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:25,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:26,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:28,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:30,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:31,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:35,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:37,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:38,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:40,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:42,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:46,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:22:46,004 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:22:46,005 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:22:46,007 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:22:46,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:46,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:22:47,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:48,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:48,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:51,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:53,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:22:53,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:22:56,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:22:57,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:22:58,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:02,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:03,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:08,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:09,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:23:09,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:13,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:14,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:18,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:19,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:23:20,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:23,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:23:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:27,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:23:29,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:30,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:36,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:37,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:23:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:41,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:45,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:46,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:47,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:23:50,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:51,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:23:53,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:23:55,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:23:57,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:00,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:02,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:05,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:07,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:10,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:12,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:15,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:17,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:20,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:22,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:23,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:24:25,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:28,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:30,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:32,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:24:33,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:35,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:38,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:41,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:43,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:44,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:24:46,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:48,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:51,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:53,344 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:24:54,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:24:54,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:24:56,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:24:59,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:02,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:04,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:05,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:25:07,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:09,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:12,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:14,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:15,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:25:17,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:19,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:23,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:23,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:23,344 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:25:25,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:25:25,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:28,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:30,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:34,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:35,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:25:36,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:39,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:41,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:44,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:45,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:25:47,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:51,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:52,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:25:53,345 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:25:55,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:25:56,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:25:57,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:01,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:02,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:05,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:26:06,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:08,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:13,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:17,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:26:18,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:23,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:26:24,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:24,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:24,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:26,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:26:29,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:30,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:34,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:35,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:36,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:26:39,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:40,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:44,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:45,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:48,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:26:48,057 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:26:48,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:26:48,059 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:26:48,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:26:48,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:26:49,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:50,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:26:51,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:26:53,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:26:54,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:26:56,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:00,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:00,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:27:01,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:05,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:06,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:10,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:10,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:10,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:27:11,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:15,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:16,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:20,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:20,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:27:22,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:23,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:27:25,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:27,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:31,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:27:31,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:33,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:36,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:38,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:41,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:41,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:44,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:45,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:27:46,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:49,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:52,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:53,348 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:27:54,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:27:55,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:27:57,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:27:59,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:02,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:04,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:05,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:28:07,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:09,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:13,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:14,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:15,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:28:18,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:19,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:23,349 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:28:23,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:23,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:24,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:25,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:28:28,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:29,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:34,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:35,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:35,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:28:39,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:40,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:44,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:45,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:47,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:28:49,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:50,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:53,350 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:28:55,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:28:56,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:28:59,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:29:00,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:01,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:05,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:06,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:08,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:29:10,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:12,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:15,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:17,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:18,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:29:20,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:22,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:23,350 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:29:26,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:26,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:27,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:28,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:29:31,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:32,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:36,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:37,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:38,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:29:41,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:42,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:46,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:47,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:51,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:52,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:29:53,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:29:55,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:29:56,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:29:58,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:02,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:03,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:05,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:30:07,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:08,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:13,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:30:14,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:18,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:19,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:22,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:30:22,176 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:30:22,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:30:22,180 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:30:22,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:30:23,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:30:23,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:30:23,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:24,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:30:24,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:28,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:29,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:34,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:34,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:35,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:30:39,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:39,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:44,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:45,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:45,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:30:49,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:50,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:53,352 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:30:55,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:30:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:30:57,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:31:00,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:00,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:05,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:06,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:07,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:31:10,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:11,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:15,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:16,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:17,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:31:20,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:21,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:23,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:31:26,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:27,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:31:31,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:32,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:36,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:37,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:41,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:31:42,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:42,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:47,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:47,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:51,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:31:52,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:53,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:31:53,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:31:58,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:31:58,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:00,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:32:03,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:08,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:08,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:11,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:32:13,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:14,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:18,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:20,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:21,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:32:23,355 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:32:23,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:25,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:29,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:30,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:33,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:32:34,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:35,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:40,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:41,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:32:41,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:45,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:46,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:51,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:51,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:32:52,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:32:53,355 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:32:56,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:32:58,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:01,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:01,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:04,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:07,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:09,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:11,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:12,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:12,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:14,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:17,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:19,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:23,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:23,357 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:33:23,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:25,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:28,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:30,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:33,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:35,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:35,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:38,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:40,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:44,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:45,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:46,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:49,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:51,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:53,358 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:33:53,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:33:53,811 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:33:53,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:33:53,815 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:33:53,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:33:54,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:54,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:33:54,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:55,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:33:56,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:33:59,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:01,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:05,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:05,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:34:07,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:10,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:12,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:16,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:17,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:34:18,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:21,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:23,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:23,360 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:34:26,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:28,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:34:28,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:31,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:33,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:36,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:34:37,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:39,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:43,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:45,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:34:45,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:48,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:50,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:53,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:34:53,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:34:55,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:34:57,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:34:58,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:00,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:03,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:06,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:07,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:35:08,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:11,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:15,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:17,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:17,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:35:20,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:22,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:23,362 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:35:25,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:27,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:35:27,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:30,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:32,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:36,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:37,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:35:38,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:41,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:43,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:46,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:47,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:35:48,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:51,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:53,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:35:54,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:35:57,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:35:57,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:35:59,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:02,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:04,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:07,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:36:08,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:10,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:13,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:15,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:19,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:19,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:36:21,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:23,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:36:24,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:27,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:29,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:29,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:29,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:36:32,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:34,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:37,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:39,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:43,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:44,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:36:44,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:48,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:53,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:36:53,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:36:54,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:36:56,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:36:58,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:01,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:04,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:04,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:37:06,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:09,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:12,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:14,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:37:14,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:17,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:19,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:23,365 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:37:23,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:23,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:37:23,883 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:37:23,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:37:23,886 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:37:24,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:37:24,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:25,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:37:28,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:29,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:34,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:35,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:37:35,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:39,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:40,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:45,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:45,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:37:46,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:50,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:51,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:37:53,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:37:55,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:37:55,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:37:57,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:00,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:02,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:05,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:05,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:38:07,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:10,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:12,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:15,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:15,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:15,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:38:17,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:20,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:22,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:23,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:38:25,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:38:26,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:28,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:31,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:33,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:36,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:37,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:38:39,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:41,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:44,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:47,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:47,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:38:49,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:52,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:53,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:38:55,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:38:57,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:38:58,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:39:00,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:02,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:06,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:08,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:39:08,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:11,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:14,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:14,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:14,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:17,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:18,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:39:19,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:22,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:23,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:39:24,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:26,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:39:28,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:29,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:33,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:34,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:38,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:42,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:39:43,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:45,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:48,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:51,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:53,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:39:53,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:39:54,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:39:56,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:39:58,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:01,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:01,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:04,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:04,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:06,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:09,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:12,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:13,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:14,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:17,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:19,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:22,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:23,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:40:23,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:25,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:27,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:32,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:32,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:35,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:36,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:37,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:41,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:43,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:45,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:46,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:48,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:51,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:53,371 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:40:53,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:40:54,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:40:54,915 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:40:54,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:40:54,918 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:40:55,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:40:55,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:56,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:40:56,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:40:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:01,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:04,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:04,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:05,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:41:06,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:09,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:11,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:14,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:16,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:41:17,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:19,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:22,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:23,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:41:25,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:26,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:41:27,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:30,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:32,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:37,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:41,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:42,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:44,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:41:46,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:47,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:51,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:52,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:41:53,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:41:54,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:41:56,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:41:58,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:01,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:04,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:42:06,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:09,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:12,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:14,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:16,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:42:17,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:19,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:22,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:23,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:42:24,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:26,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:42:27,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:29,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:33,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:33,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:35,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:36,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:42:38,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:40,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:43,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:45,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:42:45,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:48,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:50,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:53,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:42:53,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:42:56,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:42:58,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:03,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:43:04,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:04,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:06,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:09,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:13,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:43:15,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:17,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:21,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:21,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:22,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:23,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:43:24,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:43:26,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:28,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:31,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:31,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:31,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:34,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:34,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:43:36,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:39,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:42,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:44,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:46,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:43:47,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:49,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:52,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:53,375 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:43:54,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:43:56,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:43:57,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:43:59,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:02,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:04,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:07,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:12,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:44:12,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:15,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:17,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:20,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:23,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:23,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:44:24,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:44:25,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:28,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:30,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:33,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:34,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:44:35,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:39,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:40,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:43,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:44:43,079 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:44:43,080 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:44:43,083 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:44:43,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:44:44,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:44,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:44:45,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:44:46,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:49,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:51,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:53,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:44:53,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:44:54,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:44:56,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:44:59,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:01,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:05,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:06,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:45:06,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:10,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:11,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:15,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:16,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:45:16,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:20,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:21,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:23,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:45:25,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:26,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:45:27,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:30,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:32,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:36,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:37,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:38,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:45:41,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:42,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:47,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:47,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:50,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:45:52,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:52,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:45:53,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:45:58,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:45:58,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:00,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:46:03,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:03,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:08,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:08,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:08,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:10,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:46:13,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:14,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:19,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:19,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:20,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:46:23,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:46:24,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:24,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:29,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:29,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:30,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:46:34,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:34,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:39,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:39,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:44,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:46,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:46:49,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:50,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:53,381 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:46:55,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:46:55,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:46:58,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:47:00,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:01,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:06,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:06,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:06,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:09,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:47:11,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:16,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:16,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:18,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:47:21,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:22,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:23,380 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:47:27,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:27,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:28,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:47:32,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:32,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:37,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:38,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:38,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:40,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:47:43,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:43,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:48,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:48,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:50,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:47:53,391 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:47:53,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:47:53,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:47:58,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:00,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:04,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:04,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:05,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:09,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:11,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:14,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:14,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:16,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:19,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:22,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:22,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:22,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:23,381 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:48:23,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:48:23,774 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:48:23,775 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:48:23,778 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:48:23,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:48:24,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:25,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:25,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:27,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:30,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:32,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:36,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:36,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:37,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:41,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:42,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:46,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:48,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:49,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:51,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:53,382 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:48:53,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:48:56,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:48:59,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:48:59,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:01,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:06,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:06,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:08,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:49:11,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:12,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:16,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:17,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:18,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:49:21,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:22,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:23,382 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:49:27,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:28,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:30,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:49:32,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:33,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:38,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:39,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:39,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:42,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:49:43,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:44,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:48,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:50,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:53,387 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:49:54,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:49:54,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:49:55,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:49:59,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:00,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:04,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:04,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:50:06,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:09,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:11,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:14,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:16,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:17,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:50:20,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:21,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:23,389 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:50:25,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:26,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:26,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:29,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:50:30,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:31,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:35,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:37,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:37,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:37,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:40,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:41,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:50:42,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:50,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:52,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:50:53,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:50:53,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:50:56,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:50:58,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:01,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:03,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:06,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:08,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:11,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:12,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:51:13,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:17,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:18,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:22,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:23,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:51:23,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:24,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:51:27,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:28,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:32,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:34,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:36,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:51:38,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:39,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:43,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:45,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:48,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:49,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:51:50,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:53,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:51:54,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:51:55,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:55,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:51:59,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:00,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:00,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:05,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:06,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:09,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:10,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:11,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:15,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:16,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:18,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:52:18,167 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:52:18,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:52:18,170 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:52:18,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:52:19,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:20,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:21,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:21,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:23,391 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:52:26,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:27,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:31,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:31,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:32,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:36,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:37,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:41,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:42,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:43,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:46,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:47,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:51,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:52,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:52:53,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:52:53,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:52:57,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:52:58,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:02,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:03,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:07,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:08,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:08,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:53:13,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:13,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:18,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:18,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:20,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:53:23,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:53:23,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:23,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:28,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:28,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:30,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:53:33,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:33,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:38,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:39,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:39,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:43,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:53:44,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:44,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:49,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:53:49,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:53,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:53:53,393 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:53:54,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:53:55,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:00,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:00,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:03,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:54:05,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:06,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:10,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:10,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:11,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:13,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:54:15,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:16,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:20,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:21,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:23,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:54:23,393 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:54:25,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:27,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:32,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:32,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:35,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:54:37,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:37,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:42,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:42,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:45,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:54:47,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:48,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:53,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:54:53,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:53,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:53,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:53,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:54:57,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:54:58,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:54:58,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:03,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:03,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:07,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:55:08,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:09,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:14,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:14,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:16,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:55:19,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:20,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:23,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:55:24,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:25,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:28,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:55:29,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:31,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:34,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:36,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:38,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:55:40,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:41,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:45,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:46,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:50,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:51,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:55:51,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:53,396 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:55:55,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:55:57,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:55:59,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:55:59,081 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:55:59,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:55:59,086 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:55:59,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:56:00,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:00,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:01,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:03,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:05,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:08,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:11,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:11,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:13,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:16,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:18,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:22,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:23,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:23,396 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:56:24,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:27,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:29,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:32,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:33,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:34,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:37,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:39,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:42,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:45,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:45,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:47,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:50,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:53,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:56:53,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:56:55,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:56:55,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:56:58,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:00,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:03,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:05,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:57:06,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:08,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:11,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:15,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:16,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:17,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:57:20,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:21,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:23,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:57:25,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:27,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:31,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:57:31,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:32,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:36,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:37,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:41,105 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:57:41,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:41,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:43,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:46,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:48,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:51,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:57:52,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:53,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:57:53,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:57:57,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:57:58,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:02,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:03,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:58:03,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:07,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:08,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:13,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:13,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:13,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:58:14,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:18,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:19,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:23,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:58:23,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:25,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:25,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:58:28,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:30,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:34,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:35,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:37,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:58:39,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:40,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:44,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:45,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:47,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:58:49,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:51,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:53,399 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:58:56,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:58:56,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:58:59,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:01,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:01,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:06,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:06,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:09,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:11,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:11,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:12,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:17,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:18,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:22,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:22,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:23,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:23,399 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:59:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:28,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:33,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:33,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:33,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:38,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:38,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:43,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 21:59:43,349 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 21:59:43,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 21:59:43,351 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 21:59:44,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:44,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:44,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 21:59:44,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:45,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:47,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:49,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:49,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 21:59:49,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:53,400 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 21:59:54,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 21:59:55,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 21:59:59,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:00,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:04,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:05,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:09,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:10,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:14,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:15,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:19,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:20,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:23,401 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:00:24,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:26,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:29,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:31,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:34,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:36,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:39,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:41,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:44,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:46,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:49,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:51,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:53,401 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:00:54,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:00:57,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:00:59,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:02,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:04,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:07,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:09,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:12,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:14,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:17,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:19,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:22,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:23,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:01:24,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:28,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:29,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:33,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:34,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:38,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:39,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:43,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:44,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:48,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:49,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:53,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:01:54,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:54,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:01:59,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:01:59,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:04,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:04,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:09,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:09,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:14,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:14,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:19,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:19,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:23,403 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:02:24,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:25,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:29,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:30,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:35,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:39,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:40,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:44,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:45,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:49,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:50,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:53,403 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:02:54,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:02:56,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:02:59,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:01,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:04,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:06,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:09,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:11,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:14,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:16,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:19,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:21,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:23,405 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:03:24,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:27,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:29,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:32,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:34,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:37,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:39,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:44,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:47,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:49,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:52,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:53,404 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:03:54,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:03:58,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:03:59,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:03,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:08,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:09,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:13,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:14,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:18,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:19,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:23,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:04:24,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:24,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:29,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:29,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:34,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:34,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:39,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:39,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:44,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:44,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:49,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:49,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:53,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:04:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:04:55,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:04:59,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:00,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:04,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:05,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:09,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:10,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:14,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:15,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:19,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:20,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:23,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:05:24,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:26,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:29,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:31,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:34,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:36,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:39,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:41,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:44,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:46,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:49,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:51,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:53,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:05:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:05:57,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:05:59,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:02,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:04,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:07,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:09,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:12,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:14,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:19,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:22,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:23,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:06:24,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:28,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:29,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:33,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:34,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:38,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:39,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:43,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:44,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:48,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:49,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:53,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:06:54,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:54,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:06:59,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:06:59,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:04,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:04,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:09,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:09,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:14,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:14,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:19,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:19,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:23,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:07:24,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:25,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:29,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:30,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:34,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:35,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:39,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:40,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:44,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:45,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:49,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:50,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:53,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:07:54,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:07:56,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:07:59,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:01,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:04,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:06,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:09,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:11,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:14,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:16,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:19,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:21,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:23,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:08:24,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:27,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:29,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:32,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:34,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:37,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:39,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:42,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:44,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:47,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:49,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:52,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:53,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:08:54,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:08:58,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:08:59,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:03,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:04,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:08,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:09,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:13,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:14,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:18,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:19,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:23,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:09:24,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:24,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:29,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:29,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:34,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:34,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:39,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:39,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:44,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:44,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:49,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:49,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:53,411 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:09:54,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:09:55,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:09:59,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:00,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:04,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:05,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:09,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:10,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:14,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:15,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:19,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:20,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:23,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:10:24,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:26,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:29,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:31,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:36,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:39,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:41,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:44,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:46,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:49,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:51,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:53,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:10:54,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:10:57,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:10:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:02,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:04,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:07,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:12,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:14,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:17,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:19,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:22,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:23,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:11:24,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:28,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:29,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:33,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:34,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:38,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:39,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:43,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:44,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:48,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:49,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:53,413 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:11:54,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:11:54,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:11:58,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:11:58,070 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:11:58,071 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:11:58,074 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:11:58,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:11:59,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:11:59,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:00,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:00,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:12:04,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:05,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:09,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:10,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:14,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:15,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:19,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:20,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:23,414 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:12:24,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:25,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:29,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:32,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:12:34,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:36,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:40,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:40,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:41,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:46,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:46,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:51,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:52,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:53,415 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:12:56,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:12:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:12:58,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:01,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:05,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:06,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:11,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:13,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:13,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:16,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:18,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:21,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:23,416 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:13:24,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:27,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:30,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:32,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:37,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:42,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:42,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:42,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:43,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:13:44,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:13:47,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:52,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:53,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:13:57,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:58,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:58,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:13:58,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:13:59,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:14:03,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:08,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:13,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:18,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:23,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:23,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:14:28,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:32,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:14:33,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:38,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:43,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:48,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:53,418 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:14:54,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:14:59,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:04,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:09,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:10,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:15:14,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:19,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:23,428 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:15:24,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:29,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:34,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:39,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:44,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:49,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:53,419 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:15:54,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:15:54,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:15:55,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:15:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:00,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:04,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:05,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:09,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:10,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:14,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:15,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:19,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:20,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:23,420 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:16:25,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:25,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:30,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:30,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:35,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:35,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:40,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:40,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:45,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:45,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:50,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:16:50,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:53,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:16:55,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:16:56,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:00,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:01,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:05,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:06,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:08,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:17:10,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:11,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:15,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:16,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:20,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:21,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:23,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:17:27,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:32,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:36,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:36,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:36,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:37,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:39,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:17:42,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:43,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:48,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:48,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:51,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:17:53,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:17:53,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:17:53,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:58,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:17:59,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:03,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:18:04,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:04,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:09,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:09,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:14,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:19,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:18:19,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:20,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:23,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:18:25,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:26,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:26,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:29,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:18:31,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:31,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:36,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:38,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:38,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:41,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:18:41,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:43,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:46,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:49,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:51,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:53,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:18:53,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:18:54,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:18:57,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:18:59,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:02,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:04,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:07,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:09,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:12,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:14,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:19:14,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:18,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:19,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:23,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:23,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:19:25,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:26,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:19:28,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:30,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:33,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:35,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:38,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:40,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:43,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:19:44,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:45,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:49,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:52,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:53,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:19:54,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:19:55,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:19:57,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:19:59,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:03,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:04,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:07,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:20:08,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:09,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:14,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:14,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:14,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:17,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:20:19,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:19,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:23,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:20:25,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:25,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:25,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:25,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:29,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:20:31,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:31,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:36,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:38,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:41,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:20:42,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:43,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:47,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:50,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:52,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:20:53,426 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:20:53,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:20:55,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:20:57,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:01,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:02,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:05,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:21:06,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:08,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:13,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:13,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:13,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:17,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:21:18,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:18,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:23,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:23,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:21:24,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:28,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:29,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:21:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:33,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:34,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:39,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:39,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:40,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:21:40,961 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:21:40,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:21:40,971 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:21:41,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:21:42,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:21:43,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:21:44,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:44,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:49,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:49,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:53,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:21:54,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:21:55,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:21:57,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:21:59,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:00,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:05,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:05,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:07,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:22:10,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:11,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:16,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:16,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:19,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:22:21,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:21,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:23,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:22:27,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:27,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:27,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:31,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:22:32,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:33,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:37,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:38,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:42,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:42,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:43,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:43,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:22:48,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:48,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:53,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:53,433 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:22:54,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:22:57,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:22:58,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:22:59,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:03,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:04,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:08,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:09,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:09,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:23:13,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:14,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:19,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:20,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:23:20,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:23,433 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:23:24,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:26,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:29,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:31,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:23:32,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:34,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:37,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:40,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:43,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:43,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:23:45,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:48,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:50,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:53,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:23:53,436 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:23:55,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:23:56,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:23:58,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:01,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:03,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:06,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:07,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:24:09,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:11,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:14,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:17,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:24:17,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:17,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:20,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:22,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:23,435 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:24:25,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:27,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:29,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:24:31,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:32,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:36,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:38,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:41,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:24:42,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:43,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:47,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:52,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:53,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:24:53,435 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:24:56,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:24:57,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:24:58,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:02,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:03,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:07,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:08,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:25:08,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:12,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:13,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:17,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:19,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:22,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:25:23,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:23,436 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:25:24,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:28,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:30,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:33,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:25:34,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:35,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:39,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:40,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:44,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:45,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:49,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:51,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:53,437 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:25:54,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:25:56,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:25:59,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:02,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:02,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:04,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:05,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history +2023-11-20 22:26:05,737 DEBUG SenderThread:607337 [sender.py:send():380] send: history +2023-11-20 22:26:05,738 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record +2023-11-20 22:26:05,741 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end +2023-11-20 22:26:06,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json +2023-11-20 22:26:07,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:26:07,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:08,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:26:09,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:12,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:14,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:18,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:19,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:20,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:26:23,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:23,438 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:26:25,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:29,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:29,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:29,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:30,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:34,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:26:34,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:35,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:39,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:40,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:45,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:45,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:26:46,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:50,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:51,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:53,441 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:26:55,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:26:57,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:26:57,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:27:00,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:02,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:06,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:07,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:27:08,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:11,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:13,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:17,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:17,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:18,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:19,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:27:22,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:23,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:23,446 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:27:27,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:28,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:31,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:27:32,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:33,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:37,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:38,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:43,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:43,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:27:44,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:48,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:49,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:53,447 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:27:53,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:27:53,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:27:55,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:27:58,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:00,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:03,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:06,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:28:06,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:08,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:11,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:14,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:16,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:18,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:28:19,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:21,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:23,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats +2023-11-20 22:28:25,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:27,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:30,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:30,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +2023-11-20 22:28:32,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:35,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive +2023-11-20 22:28:37,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report +2023-11-20 22:28:40,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive