diff --git "a/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" "b/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" --- "a/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" +++ "b/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" @@ -1006,3 +1006,2615 @@ 2024-02-01 18:17:10,914 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log 2024-02-01 18:17:11,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report 2024-02-01 18:17:13,918 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:16,656 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:17,923 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:20,631 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:17:21,928 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:22,631 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:25,386 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:17:25,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:17:25,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:17:25,933 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:27,935 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:28,417 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:31,941 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:34,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:35,954 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:39,192 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:39,960 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:40,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:17:40,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:17:40,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:17:43,965 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:44,783 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:45,380 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:17:45,382 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:17:45,382 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:17:45,384 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:17:45,971 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:45,971 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:17:46,972 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:49,974 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:49,976 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:50,688 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:17:53,981 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:55,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:17:55,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:17:55,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:17:55,470 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:17:57,987 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:17:59,989 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:00,704 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:03,994 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:06,289 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:08,001 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:10,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:18:10,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:18:10,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:18:11,460 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:12,007 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:16,012 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:17,044 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:18,014 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:20,700 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:18:21,215 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:18:21,216 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:18:21,217 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:18:21,219 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:18:22,021 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:22,021 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:18:22,219 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:23,022 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:25,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:18:25,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:18:25,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:18:26,026 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:27,503 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:30,090 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:32,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:34,244 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:36,247 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:38,542 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:40,252 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:40,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:18:40,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:18:40,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:18:43,713 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:44,257 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:48,262 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:49,300 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:50,692 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:18:52,267 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:54,270 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:54,463 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:18:55,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:18:55,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:18:55,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:18:57,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:18:57,096 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:18:57,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:18:57,098 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:18:57,275 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:18:58,276 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:18:59,277 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:00,098 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:02,281 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:05,229 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:06,286 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:10,291 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:10,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:19:10,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:19:10,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:19:10,533 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:12,293 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:15,971 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:16,299 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:20,304 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:20,694 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:19:21,695 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:24,308 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:25,390 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:19:25,390 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:19:25,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:19:26,311 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:26,712 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:30,316 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:32,294 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:32,889 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:19:32,891 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:19:32,891 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:19:32,894 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:19:33,321 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:19:34,322 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:35,324 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:37,473 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:38,328 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:40,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:19:40,387 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:19:40,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:19:42,333 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:42,531 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:44,335 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:48,209 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:48,341 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:50,696 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:19:52,346 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:53,789 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:19:55,388 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:19:55,388 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:19:55,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:19:56,351 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:19:58,956 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:00,356 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:02,359 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:04,540 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:06,364 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:08,706 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:20:08,708 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:20:08,708 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:20:08,710 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:20:09,368 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:20:09,711 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:10,370 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:10,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:20:10,388 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:20:10,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:20:11,371 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:14,375 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:15,288 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:18,380 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:20,382 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:20,461 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:20,708 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:20:24,387 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:25,388 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:20:25,388 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:20:25,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:20:25,474 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:28,393 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:31,209 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:32,398 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:34,401 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:36,784 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:38,406 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:40,388 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:20:40,388 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:20:40,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:20:41,945 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:42,411 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:44,531 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:20:44,532 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:20:44,533 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:20:44,534 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:20:45,416 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:20:46,417 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:47,418 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:47,535 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:50,422 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:50,700 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:20:52,425 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:52,695 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:20:55,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:20:55,393 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:20:55,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:20:56,430 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:20:58,547 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:00,436 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:04,441 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:04,452 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:08,446 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:09,607 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:10,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:21:10,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:21:10,410 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:21:10,449 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:14,454 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:15,181 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:18,459 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:20,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:20,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:21:20,353 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:21:20,353 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:21:20,355 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:21:20,463 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:21:20,702 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:21:22,465 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:23,466 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:25,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:25,394 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:21:25,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:21:25,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:21:26,470 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:28,473 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:30,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:32,478 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:35,677 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:36,483 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:40,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:21:40,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:21:40,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:21:40,488 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:41,511 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:44,493 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:46,495 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:47,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:50,500 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:50,703 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:21:52,611 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:54,505 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:55,394 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:21:55,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:21:55,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:21:56,230 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:21:56,232 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:21:56,232 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:21:56,234 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:21:56,509 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:21:58,234 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:21:58,511 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:21:59,512 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:00,514 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:03,369 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:04,519 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:08,523 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:08,948 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:10,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:22:10,395 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:22:10,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:22:12,529 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:14,121 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:16,534 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:18,537 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:19,705 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:20,705 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:22:22,542 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:24,862 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:25,394 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:22:25,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:22:25,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:22:26,548 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:30,444 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:30,553 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:32,033 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:22:32,034 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:22:32,035 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:22:32,037 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:22:32,557 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:32,557 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:22:34,559 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:35,613 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:36,562 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:40,394 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:22:40,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:22:40,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:22:40,567 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:41,514 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:44,572 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:47,364 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:48,577 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:50,707 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:22:52,582 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:52,708 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:54,585 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:22:55,394 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:22:55,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:22:55,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:22:58,114 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:22:58,590 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:02,595 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:03,692 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:06,600 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:07,856 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:23:07,858 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:23:07,858 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:23:07,860 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:23:08,603 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:08,603 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:23:08,860 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:10,394 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:23:10,394 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:23:10,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:23:10,606 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:12,608 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:14,444 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:16,613 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:19,608 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:20,618 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:20,709 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:23:24,623 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:25,193 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:25,396 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:23:25,396 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:23:25,438 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:23:26,626 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:30,378 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:30,631 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:34,636 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:35,958 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:38,640 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:40,395 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:23:40,395 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:23:40,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:23:41,485 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:42,645 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:43,738 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:23:43,740 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:23:43,740 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:23:43,742 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:23:44,649 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:44,649 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:23:46,742 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:48,654 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:50,711 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:23:51,864 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:23:52,659 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:55,395 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:23:55,395 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:23:55,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:23:56,664 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:23:57,475 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:00,669 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:02,609 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:02,672 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:06,677 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:08,194 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:10,395 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:24:10,395 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:24:10,411 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:24:10,682 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:13,362 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:14,687 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:18,692 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:18,944 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:19,519 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:24:19,520 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:24:19,521 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:24:19,523 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:24:19,694 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:24:20,695 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:20,726 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:24:24,109 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:24,701 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:25,395 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:24:25,395 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:24:25,412 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:24:28,706 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:29,699 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:32,711 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:34,864 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:36,716 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:38,718 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:40,395 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:40,395 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:24:40,395 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:24:40,412 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:24:42,723 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:45,608 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:46,728 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:50,715 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:24:50,716 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:50,733 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:52,736 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:24:55,348 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:24:55,349 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:24:55,349 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:24:55,351 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:24:55,456 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:24:55,456 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:24:55,457 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:24:55,741 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:24:56,597 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:24:56,742 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:00,747 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:01,927 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:04,752 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:07,088 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:08,757 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:10,419 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:25:10,419 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:25:10,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:25:10,760 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:12,538 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:14,765 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:17,842 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:18,770 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:20,717 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:25:22,775 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:23,718 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:25,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:25:25,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:25:25,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:25:26,780 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:28,783 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:28,831 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:31,596 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:25:31,597 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:25:31,598 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:25:31,600 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:25:31,787 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:25:32,789 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:34,600 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:36,794 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:40,125 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:40,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:25:40,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:25:40,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:25:40,799 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:44,804 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:45,852 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:48,808 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:50,718 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:25:51,719 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:52,813 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:55,419 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:25:55,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:25:55,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:25:56,818 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:25:57,574 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:25:58,821 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:02,826 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:03,307 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:06,831 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:08,641 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:08,642 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:26:08,643 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:26:08,644 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:26:08,646 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:26:08,835 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:26:10,419 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:26:10,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:26:10,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:26:10,838 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:14,312 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:14,844 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:18,849 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:19,644 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:20,721 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:26:20,852 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:24,856 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:25,287 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:25,419 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:26:25,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:26:25,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:26:28,861 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:30,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:32,866 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:36,223 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:36,872 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:38,874 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:40,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:26:40,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:26:40,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:26:41,509 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:42,879 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:45,159 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:26:45,160 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:26:45,160 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:26:45,163 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:26:45,884 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:26:46,886 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:47,164 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:50,722 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:26:50,891 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:52,414 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:54,895 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:26:55,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:26:55,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:26:55,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:26:58,030 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:26:58,900 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:00,903 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:03,301 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:04,908 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:07,912 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:08,914 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:10,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:27:10,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:27:10,420 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:27:11,917 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:14,164 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:15,922 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:17,925 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:19,781 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:20,724 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:27:21,400 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:27:21,402 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:27:21,402 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:27:21,404 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:27:21,931 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:21,931 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:27:22,932 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:25,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:25,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:27:25,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:27:25,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:27:25,936 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:29,941 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:30,648 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:33,946 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:35,894 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:37,955 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:39,957 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:40,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:27:40,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:27:40,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:27:41,476 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:43,962 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:46,792 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:47,967 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:50,726 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:27:51,973 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:52,727 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:55,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:27:55,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:27:55,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:27:55,978 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:57,670 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:27:57,672 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:27:57,672 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:27:57,673 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:27:57,981 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:27:57,981 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:27:58,674 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:27:58,983 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:01,987 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:04,259 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:05,992 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:09,489 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:09,997 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:10,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:28:10,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:28:10,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:28:14,002 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:15,096 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:16,005 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:20,010 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:20,336 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:20,728 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:28:24,015 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:25,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:25,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:28:25,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:28:25,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:28:28,020 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:31,178 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:32,122 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:33,791 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:28:33,793 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:28:33,793 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:28:33,795 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:28:34,126 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:34,126 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:28:35,127 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:36,822 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:38,131 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:40,420 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:28:40,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:28:40,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:28:42,030 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:42,136 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:46,141 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:48,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:50,146 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:50,730 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:28:53,252 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:28:54,151 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:55,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:28:55,421 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:28:55,421 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:28:56,154 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:28:58,872 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:00,159 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:04,089 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:04,164 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:08,170 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:09,700 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:10,338 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:29:10,340 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:29:10,340 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:29:10,342 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:29:10,470 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:29:10,470 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:29:10,470 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:29:11,174 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:29:12,176 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:13,177 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:14,947 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:16,181 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:18,183 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:20,565 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:20,732 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:29:22,188 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:25,444 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:29:25,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:29:25,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:29:25,587 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:26,194 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:30,199 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:31,480 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:34,204 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:36,206 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:36,701 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:40,211 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:40,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:29:40,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:29:40,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:29:42,594 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:44,217 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:46,707 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:29:46,709 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:29:46,709 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:29:46,712 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:29:47,222 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:29:47,713 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:48,223 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:49,224 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:50,734 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:29:52,228 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:52,735 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:29:55,444 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:29:55,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:29:55,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:29:56,233 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:58,236 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:29:58,701 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:02,241 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:04,313 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:06,246 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:09,523 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:10,251 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:10,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:30:10,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:30:10,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:30:14,256 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:15,124 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:16,259 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:20,264 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:20,359 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:20,736 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:30:22,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:30:22,982 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:30:22,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:30:22,984 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:30:23,268 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:30:24,270 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:25,271 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:25,433 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:25,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:30:25,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:30:25,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:30:28,275 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:31,202 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:32,280 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:34,283 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:36,803 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:38,288 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:40,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:30:40,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:30:40,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:30:42,029 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:42,293 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:46,298 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:47,652 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:50,303 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:50,738 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:30:52,306 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:52,871 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:55,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:30:55,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:30:55,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:30:56,311 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:30:58,502 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:30:59,134 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:30:59,135 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:30:59,136 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:30:59,137 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:30:59,316 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:31:00,317 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:01,318 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:03,699 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:04,323 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:08,328 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:09,305 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:10,330 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:10,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:31:10,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:31:10,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:31:14,335 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:14,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:18,340 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:20,123 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:20,740 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:31:22,346 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:25,334 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:25,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:31:25,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:31:25,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:31:26,351 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:28,353 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:30,945 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:32,359 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:35,155 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:31:35,157 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:31:35,157 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:31:35,159 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:31:35,364 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:31:36,159 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:36,365 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:37,366 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:40,370 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:40,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:31:40,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:31:40,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:31:41,558 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:44,375 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:46,969 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:48,380 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:50,383 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:50,741 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:31:52,742 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:54,388 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:31:55,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:31:55,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:31:55,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:31:57,783 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:31:58,393 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:02,398 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:03,389 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:06,404 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:08,406 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:08,594 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:10,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:32:10,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:32:10,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:32:11,199 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:32:11,200 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:32:11,201 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:32:11,203 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:32:11,411 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:32:12,413 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:13,414 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:14,203 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:16,418 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:19,399 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:20,423 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:20,743 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:32:24,429 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:25,000 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:25,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:32:25,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:32:25,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:32:26,431 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:30,206 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:30,436 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:34,441 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:35,644 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:36,444 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:38,447 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:40,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:32:40,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:32:40,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:32:40,449 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:40,653 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:42,452 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:44,454 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:45,658 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:46,457 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:48,460 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:50,462 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:50,673 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:50,745 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:32:52,465 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:54,467 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:55,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:32:55,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:32:55,446 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:32:55,676 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:32:56,470 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:32:58,472 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:00,475 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:00,686 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:02,478 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:03,268 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:33:03,270 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:33:03,271 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:33:03,273 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:33:03,480 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:33:04,481 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:05,482 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:06,374 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:06,484 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:08,487 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:09,489 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:11,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:11,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:33:11,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:33:11,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:33:13,494 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:16,498 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:17,054 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:18,501 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:20,747 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:33:22,506 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:22,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:25,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:33:26,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:33:26,464 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:33:26,511 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:27,608 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:28,697 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:33:28,699 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:33:28,699 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:33:28,701 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:33:29,515 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:33:30,517 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:31,518 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:33,287 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:34,521 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:36,524 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:38,863 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:40,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:33:40,530 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:41,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:33:41,464 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:33:44,054 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:44,535 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:48,540 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:49,663 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:50,748 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:33:52,545 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:54,548 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:33:54,849 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:33:55,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:33:56,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:33:56,464 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:33:58,553 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:00,437 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:02,557 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:04,632 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:34:04,633 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:34:04,634 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:34:04,635 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:34:05,562 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:34:05,636 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:06,564 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:10,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:34:10,569 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:11,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:34:11,464 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:34:11,607 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:12,571 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:16,576 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:17,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:20,581 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:20,751 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:34:22,592 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:24,587 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:25,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:34:26,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:34:26,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:34:28,547 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:28,591 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:30,594 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:34,364 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:34,599 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:38,604 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:39,961 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:40,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:34:40,590 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:34:40,592 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:34:40,592 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:34:40,594 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:34:40,607 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:34:41,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:34:41,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:34:42,610 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:45,137 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:46,615 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:48,617 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:50,730 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:50,763 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:34:52,622 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:34:55,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:34:56,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:34:56,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:34:56,527 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:34:56,627 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:00,632 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:02,097 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:04,638 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:06,640 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:07,681 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:10,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:35:10,645 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:11,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:35:11,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:35:12,870 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:14,650 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:16,461 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:35:16,463 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:35:16,463 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:35:16,465 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:35:16,654 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:35:18,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:18,656 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:20,755 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:35:22,662 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:23,647 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:24,664 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:25,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:35:26,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:35:26,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:35:28,669 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:29,235 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:32,674 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:34,423 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:36,679 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:38,682 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:40,016 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:40,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:35:41,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:35:41,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:35:42,690 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:45,194 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:46,692 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:50,697 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:50,757 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:35:50,757 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:52,369 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:35:52,371 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:35:52,371 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:35:52,374 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:35:52,701 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:35:54,704 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:35:55,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:35:55,971 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:35:56,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:35:56,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:35:56,706 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:00,711 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:01,558 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:04,717 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:06,742 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:08,723 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:10,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:36:11,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:36:11,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:36:12,590 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:12,728 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:14,732 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:18,497 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:18,736 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:20,763 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:36:22,741 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:24,089 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:25,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:36:26,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:36:26,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:36:26,746 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:28,265 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:36:28,267 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:36:28,267 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:36:28,269 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:36:28,750 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:36:29,269 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:30,752 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:32,755 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:34,854 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:36,760 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:40,028 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:40,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:36:40,765 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:41,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:36:41,465 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:36:44,770 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:45,620 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:48,775 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:50,777 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:36:50,778 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:50,779 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:54,783 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:36:55,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:36:56,465 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:36:56,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:36:56,588 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:36:58,788 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:02,574 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:02,794 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:04,155 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:37:04,156 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:37:04,157 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:37:04,159 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:37:04,797 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:04,797 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:37:06,800 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:07,734 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:08,802 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:10,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:37:11,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:37:11,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:37:12,807 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:13,596 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:16,813 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:19,499 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:20,767 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:37:20,818 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:24,823 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:25,090 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:25,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:37:26,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:37:26,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:37:26,825 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:30,283 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:30,831 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:34,836 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:35,870 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:38,910 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:40,064 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:37:40,065 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:37:40,066 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:37:40,068 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:37:40,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:37:40,914 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:40,914 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:37:41,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:37:41,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:37:41,602 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:42,916 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:44,919 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:46,651 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:48,925 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:50,769 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:37:51,832 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:52,931 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:55,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:37:56,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:37:56,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:37:56,936 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:37:57,521 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:37:58,938 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:02,610 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:02,943 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:06,948 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:08,209 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:10,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:38:10,953 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:11,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:38:11,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:38:13,379 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:14,958 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:15,972 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:38:15,973 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:38:15,974 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:38:15,976 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:38:16,962 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:16,963 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:38:18,976 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:20,771 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:38:20,967 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:24,159 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:24,972 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:25,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:38:26,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:38:26,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:38:28,977 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:29,754 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:32,983 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:34,116 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:34,929 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:38,122 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:40,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:38:40,518 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:41,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:38:41,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:38:42,135 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:45,706 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:46,140 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:50,145 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:50,783 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:38:50,785 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:38:51,903 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:38:51,904 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:38:51,905 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:38:51,906 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:38:52,150 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:52,150 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:38:55,603 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:38:56,155 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:38:56,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:38:56,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:38:56,555 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:00,160 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:02,089 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:04,165 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:07,285 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:08,171 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:10,173 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:10,567 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:39:11,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:39:11,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:39:12,551 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:14,179 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:18,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:18,185 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:20,775 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:39:22,190 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:23,777 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:25,567 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:39:26,195 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:26,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:39:26,466 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:39:27,878 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:39:27,879 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:39:27,879 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:39:27,881 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:39:28,198 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:28,199 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:39:28,881 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:32,203 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:34,463 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:36,209 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:39,670 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:40,214 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:40,567 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:39:41,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:39:41,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:39:44,220 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:45,260 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:46,223 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:50,228 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:50,443 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:50,777 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:39:54,233 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:39:55,567 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:39:55,580 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:39:56,466 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:39:56,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:39:58,238 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:01,216 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:02,243 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:03,811 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:40:03,812 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:40:03,812 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:40:03,816 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:40:04,247 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:04,247 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:40:06,817 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:08,252 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:10,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:40:11,467 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:40:11,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:40:12,257 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:12,608 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:16,264 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:18,187 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:20,268 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:20,779 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:40:22,271 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:23,778 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:25,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:40:26,276 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:26,467 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:40:26,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:40:28,954 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:30,281 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:34,286 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:34,552 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:38,290 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:39,732 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:39,734 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:40:39,735 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:40:39,735 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:40:39,737 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:40:40,294 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:40,294 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:40:40,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:40:41,467 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:40:41,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:40:44,299 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:45,324 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:48,304 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:50,503 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:50,791 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:40:52,309 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:54,312 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:40:55,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:40:55,580 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:40:56,467 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:40:56,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:40:58,317 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:00,678 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:02,321 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:05,847 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:06,327 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:10,332 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:10,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:41:11,467 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:41:11,467 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:41:11,510 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:12,334 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:15,631 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:41:15,632 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:41:15,633 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:41:15,635 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:41:16,340 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:16,340 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:41:16,635 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:20,345 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:20,783 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:41:21,784 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:24,350 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:25,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:41:26,516 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:41:26,517 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:41:27,590 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:28,355 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:30,357 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:33,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:34,362 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:38,205 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:38,367 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:40,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:41:41,480 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:41:41,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:41:42,371 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:43,594 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:46,376 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:48,378 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:48,991 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:50,785 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:41:51,589 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:41:51,590 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:41:51,590 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:41:51,593 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:41:52,384 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:52,385 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:41:54,594 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:41:55,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:41:56,389 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:41:56,480 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:41:56,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:41:59,764 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:00,394 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:04,399 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:05,359 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:06,402 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:10,407 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:10,527 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:10,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:42:11,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:42:11,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:42:14,412 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:16,111 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:18,417 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:20,787 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:42:21,788 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:22,421 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:24,424 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:25,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:42:26,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:42:26,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:42:27,477 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:27,478 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:42:27,479 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:42:27,480 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:42:27,482 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:42:28,429 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:28,429 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:42:32,434 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:33,164 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:36,459 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:38,507 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:40,464 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:40,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:42:41,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:42:41,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:42:44,160 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:44,469 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:46,472 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:49,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:50,477 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:50,799 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:42:54,482 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:42:55,108 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:42:55,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:42:56,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:42:56,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:42:58,487 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:00,387 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:02,492 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:04,010 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:43:04,012 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:43:04,012 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:43:04,014 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:43:04,495 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:04,496 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:43:06,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:08,501 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:10,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:43:11,282 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:11,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:43:11,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:43:12,506 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:16,512 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:16,900 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:20,518 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:20,791 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:43:22,149 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:24,523 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:25,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:43:26,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:43:26,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:43:26,525 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:27,639 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:30,530 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:32,979 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:34,535 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:38,540 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:38,602 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:40,211 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:43:40,213 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:43:40,213 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:43:40,215 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:43:40,544 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:43:40,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:43:41,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:43:41,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:43:41,545 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:42,546 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:43,836 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:44,550 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:48,555 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:49,458 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:50,793 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:43:52,560 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:43:54,693 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:43:55,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:43:56,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:43:56,481 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:43:56,564 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:00,309 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:00,569 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:02,572 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:05,522 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:06,576 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:10,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:44:10,581 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:10,582 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:11,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:44:11,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:44:14,587 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:15,726 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:16,340 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:44:16,342 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:44:16,342 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:44:16,344 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:44:16,590 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:44:18,592 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:20,595 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:20,795 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:44:20,796 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:24,600 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:25,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:44:26,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:44:26,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:44:26,608 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:28,605 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:31,758 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:32,609 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:36,614 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:37,358 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:38,617 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:40,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:44:41,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:44:41,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:44:42,621 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:42,622 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:46,626 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:48,194 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:50,631 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:50,807 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:44:52,399 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:44:52,401 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:44:52,401 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:44:52,403 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:44:52,635 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:44:53,403 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:44:54,637 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:55,569 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:44:56,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:44:56,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:44:56,640 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:44:58,619 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:00,644 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:04,206 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:04,649 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:08,654 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:09,812 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:10,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:45:11,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:45:11,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:45:12,659 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:14,662 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:15,022 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:18,667 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:20,624 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:20,798 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:45:22,672 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:25,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:45:26,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:45:26,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:45:26,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:26,677 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:28,445 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:45:28,446 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:45:28,447 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:45:28,448 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:45:28,680 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:45:30,683 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:32,050 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:32,686 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:36,690 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:37,654 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:40,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:45:40,695 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:41,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:45:41,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:45:42,854 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:44,700 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:48,468 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:48,705 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:50,708 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:50,800 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:45:53,667 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:45:54,713 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:55,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:45:56,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:45:56,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:45:58,718 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:45:59,276 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:02,723 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:04,500 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:04,501 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:46:04,503 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:46:04,503 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:46:04,505 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:46:04,726 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:46:06,728 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:08,731 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:10,107 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:10,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:46:11,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:46:11,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:46:12,736 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:15,312 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:16,742 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:20,747 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:20,802 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:46:20,803 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:24,752 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:25,570 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:46:26,135 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:26,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:46:26,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:46:26,754 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:30,759 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:31,737 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:34,764 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:36,935 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:38,769 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:40,542 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:46:40,543 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:46:40,544 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:46:40,546 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:46:40,664 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:46:40,773 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:46:41,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:46:41,482 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:46:42,619 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:42,775 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:44,777 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:47,756 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:48,782 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:50,816 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:46:52,787 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:53,357 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:46:55,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:46:56,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:46:56,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:46:56,792 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:46:58,568 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:00,797 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:02,799 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:04,168 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:06,804 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:09,375 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:10,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:47:10,809 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:11,482 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:47:11,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:47:14,814 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:14,977 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:16,592 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:47:16,593 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:47:16,594 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:47:16,596 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:47:16,818 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:47:18,820 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:20,199 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:20,806 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:47:22,825 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:24,827 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:25,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:47:25,653 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:26,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:47:26,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:47:28,832 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:31,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:32,837 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:36,612 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:36,844 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:40,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:47:40,849 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:41,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:47:41,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:47:41,818 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:42,852 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:46,856 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:47,428 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:50,808 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:47:50,861 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:52,654 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:52,658 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:47:52,659 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:47:52,660 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:47:52,662 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:47:52,868 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:47:54,870 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:47:55,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:47:56,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:47:56,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:47:58,580 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:47:58,875 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:00,878 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:04,454 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:04,884 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:08,889 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:10,062 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:10,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:48:11,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:48:11,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:48:12,894 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:15,295 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:16,898 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:18,901 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:20,809 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:48:20,810 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:22,905 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:25,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:48:26,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:48:26,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:48:26,597 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:26,910 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:28,707 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:48:28,708 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:48:28,708 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:48:28,710 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:48:28,913 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:48:30,916 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:31,711 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:34,921 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:36,924 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:37,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:40,632 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:48:41,053 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:41,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:48:41,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:48:42,562 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:45,058 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:47,817 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:49,063 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:50,821 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:48:52,822 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:53,069 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:55,072 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:48:55,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:48:56,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:48:56,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:48:58,628 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:48:59,077 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:03,082 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:04,223 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:04,828 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:49:04,830 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:49:04,830 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:49:04,832 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:49:05,086 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:49:07,088 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:09,427 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:10,630 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:49:11,093 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:11,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:49:11,483 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:49:13,095 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:15,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:17,101 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:20,216 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:20,813 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:49:21,105 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:25,110 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:25,631 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:49:25,654 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:26,530 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:49:26,530 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:49:29,115 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:31,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:31,118 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:35,123 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:36,609 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:39,128 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:40,631 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:49:41,130 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:41,492 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:49:41,492 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:49:41,632 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:43,133 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:45,135 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:46,647 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:47,138 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:49,140 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:50,815 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:49:51,143 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:51,651 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:53,146 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:55,148 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:55,631 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:49:56,492 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:49:56,492 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:49:56,670 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:49:57,151 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:49:59,153 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:01,156 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:01,687 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:03,158 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:05,161 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:06,697 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:07,163 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:07,782 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:50:07,784 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:50:07,784 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:50:07,788 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:50:08,165 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:50:09,167 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:11,169 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:11,871 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:11,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:50:11,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:50:11,964 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:50:13,172 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:17,179 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:17,357 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:19,182 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:20,817 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:50:21,184 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:22,593 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:22,594 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:50:22,595 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:50:22,596 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:50:22,598 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:50:23,187 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:50:25,190 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:26,948 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:50:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:50:26,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:50:27,192 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:28,053 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:31,197 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:33,340 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:35,202 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:38,926 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:39,207 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:41,948 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:50:41,948 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:50:41,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:50:43,212 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:44,001 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:45,214 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:49,219 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:49,655 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:50,818 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:50:53,224 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:54,820 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:50:56,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:50:56,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:50:56,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:50:57,229 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:50:58,446 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:50:58,448 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:50:58,448 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:50:58,450 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:50:59,232 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:51:00,450 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:01,235 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:03,237 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:05,554 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:07,242 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:11,133 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:11,247 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:11,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:51:11,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:51:11,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:51:15,252 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:16,301 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:19,257 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:20,820 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:51:21,259 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:21,821 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:25,264 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:51:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:51:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:26,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:51:29,269 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:32,620 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:33,274 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:34,198 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:51:34,199 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:51:34,200 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:51:34,202 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:51:34,276 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:51:35,278 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:37,771 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:39,284 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:41,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:51:41,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:51:41,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:51:42,289 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:43,008 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:46,297 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:48,522 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:50,306 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:50,822 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:51:52,310 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:54,110 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:51:56,318 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:51:56,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:51:56,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:51:56,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:51:59,110 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:00,326 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:04,335 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:04,858 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:08,343 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:10,027 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:10,028 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:52:10,029 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:52:10,030 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:52:10,032 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:52:10,349 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:10,349 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:52:11,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:52:11,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:52:11,949 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:52:14,357 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:15,611 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:18,365 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:20,762 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:20,824 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:52:22,371 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:26,341 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:26,376 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:52:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:52:26,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:52:28,379 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:31,515 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:32,383 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:36,388 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:37,100 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:40,393 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:41,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:52:41,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:52:41,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:52:42,275 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:42,396 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:45,866 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:52:45,868 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:52:45,868 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:52:45,870 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:52:46,402 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:46,402 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:52:47,871 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:50,407 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:50,839 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:52:53,041 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:52:54,412 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:56,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:52:56,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:52:56,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:52:58,416 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:52:59,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:00,419 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:04,424 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:04,797 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:08,429 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:10,383 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:11,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:53:11,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:53:11,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:53:12,434 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:15,559 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:16,439 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:18,441 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:20,828 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:53:20,829 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:21,733 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:53:21,735 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:53:21,735 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:53:21,737 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:53:22,447 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:22,447 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:53:26,314 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:26,452 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:26,949 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:53:26,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:53:26,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:53:30,457 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:31,899 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:34,462 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:36,464 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:37,064 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:40,469 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:41,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:53:41,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:53:41,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:53:43,016 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:44,474 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:48,479 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:48,802 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:50,830 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:53:52,483 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:53,831 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:53:54,486 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:56,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:53:56,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:53:56,950 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:53:57,602 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:53:57,604 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:53:57,604 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:53:57,605 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:53:58,492 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:53:58,492 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:53:59,606 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:02,498 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:04,734 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:06,502 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:10,323 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:10,509 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:12,017 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:54:12,017 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:54:12,017 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:54:12,511 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:15,491 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:16,516 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:20,520 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:20,832 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:54:20,833 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:24,525 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:26,249 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:26,527 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:26,979 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:54:26,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:54:27,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:54:30,532 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:31,820 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:33,404 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:54:33,405 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:54:33,405 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:54:33,407 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:54:33,537 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:54:34,538 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:36,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:38,543 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:41,979 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:54:41,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:54:42,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:54:42,125 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:42,548 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:44,551 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:47,156 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:48,555 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:50,844 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:54:52,321 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:54:52,560 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:56,565 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:54:56,979 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:54:56,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:54:57,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:54:58,136 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:00,570 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:02,573 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:04,065 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:06,578 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:09,244 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:09,245 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:55:09,246 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:55:09,247 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:55:09,249 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:55:09,582 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:55:10,584 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:11,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:55:11,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:55:12,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:55:14,589 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:14,824 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:18,593 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:19,979 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:20,596 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:20,836 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:55:24,601 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:25,573 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:26,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:55:26,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:55:27,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:55:28,606 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:30,724 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:32,611 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:34,614 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:36,313 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:38,619 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:41,483 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:41,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:55:41,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:55:42,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:55:42,623 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:45,114 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:55:45,116 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:55:45,116 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:55:45,119 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:55:45,628 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:55:46,629 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:47,119 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:50,634 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:50,838 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:55:52,289 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:55:52,637 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:56,642 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:55:56,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:55:56,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:55:57,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:55:58,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:00,647 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:04,021 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:04,651 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:08,657 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:09,609 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:10,659 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:11,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:56:11,980 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:56:12,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:56:14,664 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:14,795 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:18,669 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:20,372 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:20,839 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:56:20,959 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:56:20,961 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:56:20,961 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:56:20,963 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:56:21,674 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:56:22,675 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:25,543 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:26,680 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:26,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:56:26,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:56:27,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:56:28,683 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:31,124 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:32,688 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:36,654 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:36,693 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:40,698 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:41,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:41,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:56:41,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:56:42,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:56:44,703 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:47,571 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:48,708 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:50,711 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:50,851 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:56:52,852 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:54,716 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:56:56,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:56:56,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:56:57,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:56:57,526 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:56:57,528 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:56:57,528 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:56:57,530 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:56:57,721 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:56:58,530 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:56:58,722 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:02,727 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:04,071 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:06,732 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:08,734 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:09,239 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:11,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:57:11,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:57:12,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:57:12,739 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:15,079 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:16,744 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:20,749 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:20,843 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:57:20,844 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:24,755 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:26,164 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:26,757 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:26,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:57:26,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:57:27,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:57:30,762 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:31,743 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:33,323 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:57:33,324 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:57:33,324 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:57:33,327 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:57:33,767 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:57:34,768 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:36,901 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:38,775 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:41,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:41,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:57:41,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:57:42,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:57:42,780 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:44,782 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:47,067 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:48,787 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:50,845 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:57:52,230 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:57:52,794 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:56,802 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:57:56,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:57:56,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:57:57,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:57:58,120 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:00,810 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:02,814 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:03,993 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:06,823 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:09,172 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:09,173 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:58:09,175 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:58:09,175 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:58:09,177 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:58:09,830 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:58:10,831 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:11,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:58:11,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:58:12,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:58:14,760 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:14,840 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:16,844 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:19,921 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:20,847 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:58:20,850 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:24,855 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:25,508 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:26,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:58:26,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:58:27,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:58:28,862 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:30,680 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:32,867 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:34,869 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:36,257 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:38,894 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:41,422 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:41,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:58:41,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:58:42,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:58:43,010 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:45,050 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:58:45,052 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:58:45,052 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:58:45,054 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:58:46,015 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:58:47,017 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:47,054 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:50,859 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:58:51,022 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:52,176 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:58:53,024 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:56,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:58:56,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:58:57,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:58:57,031 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:58:58,092 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:01,036 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:03,916 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:05,041 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:09,046 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:09,521 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:11,049 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:11,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:59:11,981 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:59:12,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:59:14,701 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:15,054 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:19,059 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:20,293 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:20,854 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:59:20,892 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:59:20,893 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:59:20,893 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:59:20,896 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:59:21,062 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:59:23,065 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:25,503 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:26,981 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:59:26,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:59:27,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:59:27,070 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:29,072 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:31,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:33,077 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:36,458 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:37,082 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:41,087 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:41,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:41,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:59:41,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:59:42,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:59:45,092 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:47,674 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:49,097 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:50,853 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 18:59:52,855 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:53,102 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:55,104 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 18:59:56,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 18:59:56,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 18:59:57,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 18:59:57,794 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 18:59:57,796 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 18:59:57,796 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 18:59:57,798 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 18:59:58,109 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 18:59:58,798 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 18:59:59,110 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:03,115 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:04,460 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:07,119 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:09,798 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:11,125 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:11,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:00:11,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:00:12,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:00:15,129 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:15,469 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:17,132 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:20,773 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:20,855 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:00:21,140 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:25,145 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:26,426 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:26,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:00:26,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:00:27,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:00:29,150 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:31,707 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:33,155 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:34,350 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:00:34,352 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:00:34,352 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:00:34,355 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:00:35,158 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:35,159 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:00:37,356 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:39,163 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:41,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:00:41,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:00:42,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:00:43,079 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:43,168 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:47,173 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:48,265 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:50,857 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:00:51,178 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:53,527 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:00:55,184 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:56,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:00:56,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:00:57,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:00:57,186 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:00:59,097 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:01,191 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:04,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:05,196 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:09,201 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:10,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:10,649 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:01:10,651 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:01:10,651 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:01:10,653 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:01:11,205 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:01:11,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:01:11,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:01:12,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:01:13,207 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:15,213 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:15,280 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:19,217 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:20,859 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:01:20,860 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:23,222 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:26,143 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:26,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:01:26,982 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:01:27,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:01:27,227 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:31,232 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:31,765 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:33,235 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:36,999 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:37,239 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:41,244 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:41,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:01:41,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:01:42,005 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:01:42,125 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:45,249 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:46,862 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:01:46,863 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:01:46,863 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:01:46,865 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:01:47,253 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:01:47,866 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:49,255 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:50,861 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:01:53,261 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:53,862 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:01:55,264 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:56,982 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:01:56,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:01:57,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:01:59,269 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:01:59,719 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:03,274 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:04,955 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:07,279 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:10,579 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:11,284 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:11,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:02:11,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:02:12,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:02:13,287 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:15,798 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:17,292 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:20,863 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:02:20,864 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:21,297 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:23,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:02:23,025 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:02:23,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:02:23,027 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:02:23,301 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:02:25,303 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:26,028 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:26,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:02:26,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:02:27,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:02:29,308 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:31,276 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:31,311 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:35,316 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:36,875 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:39,321 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:41,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:41,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:02:41,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:02:42,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:02:43,326 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:47,331 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:47,709 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:49,333 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:50,865 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:02:52,930 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:53,339 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:56,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:02:56,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:02:57,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:02:57,344 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:02:58,039 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:02:59,182 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:02:59,186 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:02:59,187 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:02:59,189 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:02:59,348 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:03:01,351 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:03,752 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:05,355 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:07,358 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:09,356 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:11,363 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:11,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:03:11,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:03:12,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:03:14,573 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:15,368 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:19,373 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:20,200 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:20,867 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:03:23,377 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:25,380 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:25,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:26,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:03:26,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:03:27,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:03:29,385 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:31,020 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:33,390 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:35,241 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:03:35,243 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:03:35,243 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:03:35,245 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:03:35,393 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:03:36,245 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:37,396 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:41,403 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:41,854 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:41,983 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:03:41,983 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:03:42,006 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:03:45,413 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:47,079 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:47,417 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:50,869 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:03:51,426 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:52,870 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:55,435 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:03:57,047 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:03:57,048 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:03:57,048 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:03:58,130 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:03:59,444 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:03,453 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:03,517 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:05,458 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:08,743 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:09,467 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:11,351 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:04:11,352 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:04:11,352 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:04:11,361 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:04:11,475 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:04:12,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:04:12,022 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:04:12,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:04:12,477 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:14,032 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:16,486 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:19,572 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:20,497 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:20,881 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:04:22,503 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:25,182 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:26,510 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:27,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:04:27,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:04:27,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:04:30,398 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:30,515 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:34,520 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:36,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:38,525 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:40,527 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:41,222 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:42,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:04:42,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:04:42,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:04:44,532 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:46,835 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:47,443 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:04:47,444 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:04:47,445 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:04:47,447 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:04:47,537 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:04:48,538 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:50,873 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:04:52,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:52,543 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:56,548 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:04:57,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:04:57,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:04:57,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:04:57,143 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:04:58,551 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:02,556 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:02,875 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:06,561 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:08,488 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:10,566 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:12,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:05:12,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:05:12,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:05:13,694 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:14,571 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:16,574 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:19,311 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:20,579 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:20,875 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:05:23,520 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:05:23,521 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:05:23,521 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:05:23,523 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:05:23,584 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:05:24,524 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:24,585 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:27,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:05:27,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:05:27,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:05:28,590 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:30,182 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:32,595 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:34,598 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:35,327 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:38,603 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:40,955 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:42,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:05:42,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:05:42,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:05:42,608 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:46,187 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:46,613 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:50,618 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:50,877 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:05:51,878 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:52,621 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:56,626 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:05:56,997 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:05:57,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:05:57,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:05:57,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:05:59,612 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:05:59,614 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:05:59,614 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:05:59,616 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:05:59,631 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:06:00,632 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:02,617 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:04,637 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:07,829 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:08,642 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:10,645 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:12,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:06:12,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:06:12,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:06:13,117 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:14,650 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:18,655 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:18,664 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:20,888 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:06:22,660 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:24,274 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:26,665 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:27,011 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:06:27,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:06:27,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:06:29,481 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:30,677 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:32,679 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:35,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:35,692 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:06:35,694 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:06:35,694 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:06:35,696 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:06:36,685 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:36,686 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:06:40,299 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:40,690 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:42,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:06:42,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:06:42,023 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:06:44,695 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:45,724 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:46,698 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:48,701 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:50,704 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:50,881 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:06:50,882 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:52,706 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:54,709 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:56,232 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:06:56,711 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:06:57,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:06:57,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:06:57,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:06:58,714 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:00,717 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:01,731 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:02,719 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:04,722 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:06,725 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:07,229 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:08,727 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:10,730 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:12,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:07:12,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:07:12,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:07:12,229 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:12,733 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:13,300 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:07:13,301 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:07:13,301 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:07:13,304 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:07:13,735 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:07:14,736 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:17,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:18,741 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:20,882 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:07:22,810 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:23,749 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:24,750 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:27,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:07:27,023 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:07:27,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:07:28,077 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:28,755 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:32,760 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:33,668 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:36,765 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:38,867 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:40,772 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:42,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:07:42,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:07:42,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:07:42,774 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:44,470 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:46,780 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:49,653 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:50,785 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:50,884 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:07:53,248 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:07:53,249 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:07:53,250 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:07:53,252 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:07:53,791 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:07:54,793 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:07:55,252 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:07:57,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:07:57,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:07:57,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:07:58,802 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:00,441 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:00,807 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:04,815 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:06,021 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:08,824 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:11,189 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:12,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:08:12,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:08:12,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:08:12,833 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:16,782 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:16,838 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:18,841 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:20,886 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:08:21,887 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:22,846 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:26,852 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:27,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:08:27,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:27,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:08:27,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:08:29,147 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:08:29,148 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:08:29,148 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:08:29,150 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:08:29,856 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:08:30,858 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:32,159 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report