diff --git "a/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" "b/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" --- "a/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" +++ "b/wandb/run-20240201_175850-i93q0p12/logs/debug-internal.log" @@ -3618,3 +3618,844 @@ 2024-02-01 19:08:29,856 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json 2024-02-01 19:08:30,858 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log 2024-02-01 19:08:32,159 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:34,863 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:36,865 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:37,330 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:40,888 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:42,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:08:42,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:08:42,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:08:43,091 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:44,992 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:48,110 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:48,997 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:50,888 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:08:53,002 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:53,889 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:55,005 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:08:57,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:08:57,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:08:57,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:08:58,897 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:08:59,010 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:03,015 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:04,497 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:05,090 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:09:05,091 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:09:05,092 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:09:05,094 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:09:06,020 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:09:07,021 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:09,689 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:11,026 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:12,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:09:12,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:09:12,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:09:13,029 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:15,282 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:17,034 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:20,464 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:20,935 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:09:21,039 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:25,044 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:26,067 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:27,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:09:27,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:09:27,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:09:29,049 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:31,051 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:31,259 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:35,056 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:36,859 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:39,065 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:41,044 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:09:41,045 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:09:41,046 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:09:41,051 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:09:41,069 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:09:42,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:09:42,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:42,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:09:42,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:09:43,074 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:47,084 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:47,636 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:49,089 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:50,937 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:09:52,809 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:09:53,099 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:57,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:09:57,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:09:57,024 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:09:57,107 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:09:58,142 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:01,117 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:03,576 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:05,126 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:07,131 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:09,158 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:11,140 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:12,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:10:12,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:10:12,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:10:14,344 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:15,147 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:16,944 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:10:16,945 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:10:16,945 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:10:16,951 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:10:17,150 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:10:19,153 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:19,952 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:20,949 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:10:21,155 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:25,138 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:25,160 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:27,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:10:27,024 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:10:27,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:10:29,165 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:30,726 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:33,171 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:35,909 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:37,176 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:39,178 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:41,496 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:42,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:10:42,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:10:42,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:10:43,183 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:47,066 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:47,188 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:50,941 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:10:51,193 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:52,942 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:53,258 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:10:53,259 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:10:53,260 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:10:53,263 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:10:54,198 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:10:55,199 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:10:57,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:10:57,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:10:57,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:10:58,142 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:10:59,204 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:01,207 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:03,449 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:05,212 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:08,632 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:09,217 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:12,013 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:11:12,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:11:12,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:11:13,222 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:14,075 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:17,227 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:19,229 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:19,408 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:20,943 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:11:23,234 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:24,994 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:27,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:11:27,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:11:27,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:11:27,239 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:29,170 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:11:29,171 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:11:29,172 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:11:29,174 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:11:29,242 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:11:30,174 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:31,244 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:35,250 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:35,767 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:37,252 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:40,959 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:41,258 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:42,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:11:42,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:11:42,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:11:45,263 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:46,558 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:49,268 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:50,945 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:11:51,946 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:53,273 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:55,276 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:11:57,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:11:57,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:11:57,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:11:57,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:11:59,281 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:02,499 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:03,286 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:05,089 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:12:05,090 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:12:05,091 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:12:05,093 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:12:05,290 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:12:07,292 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:08,093 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:11,297 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:12,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:12:12,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:12:12,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:12:13,286 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:13,300 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:17,305 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:18,888 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:20,957 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:12:21,310 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:24,074 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:25,315 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:27,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:12:27,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:12:27,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:12:29,106 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:29,320 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:31,322 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:34,865 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:35,327 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:39,332 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:40,470 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:41,067 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:12:41,069 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:12:41,069 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:12:41,071 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:12:41,336 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:12:42,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:12:42,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:12:42,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:12:43,339 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:45,663 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:47,344 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:49,346 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:50,949 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:12:50,950 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:53,351 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:12:56,459 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:12:57,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:12:57,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:12:57,025 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:12:57,356 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:01,361 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:02,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:05,366 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:07,253 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:07,369 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:11,374 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:12,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:13:12,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:13:12,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:13:13,157 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:15,379 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:17,048 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:13:17,049 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:13:17,050 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:13:17,052 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:13:17,383 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:13:19,052 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:19,385 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:20,951 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:13:23,390 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:24,255 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:25,393 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:27,014 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:13:27,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:13:27,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:13:29,398 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:29,847 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:33,403 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:35,043 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:37,408 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:40,628 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:41,413 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:42,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:13:42,026 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:13:42,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:13:43,416 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:45,820 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:47,421 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:50,953 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:13:50,954 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:51,427 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:53,002 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:13:53,004 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:13:53,004 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:13:53,007 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:13:53,431 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:13:55,433 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:13:56,008 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:13:57,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:13:57,026 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:13:57,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:13:57,437 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:01,178 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:01,442 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:05,448 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:06,770 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:09,453 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:11,940 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:12,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:14:12,026 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:14:12,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:14:13,459 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:15,461 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:17,537 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:19,466 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:20,955 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:14:22,701 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:23,471 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:27,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:14:27,026 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:14:27,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:14:27,476 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:28,130 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:28,895 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:14:28,896 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:14:28,896 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:14:28,898 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:14:29,480 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:14:31,482 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:33,471 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:33,485 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:37,490 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:39,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:41,495 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:42,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:14:42,026 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:14:42,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:14:44,243 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:45,500 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:49,505 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:49,840 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:50,957 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:14:51,507 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:55,025 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:14:55,512 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:14:57,015 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:14:57,026 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:14:57,026 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:14:59,517 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:00,619 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:03,522 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:04,801 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:15:04,803 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:15:04,803 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:15:04,805 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:15:05,526 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:15:05,806 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:07,528 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:09,531 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:11,393 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:12,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:15:12,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:15:12,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:15:13,536 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:16,561 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:17,541 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:20,959 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:15:21,546 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:21,960 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:25,551 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:27,058 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:15:27,083 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:27,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:15:27,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:15:27,554 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:31,559 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:32,930 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:35,564 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:38,099 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:39,569 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:40,687 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:15:40,688 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:15:40,688 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:15:40,690 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:15:41,572 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:15:42,058 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:15:42,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:15:42,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:15:43,183 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:43,575 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:45,578 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:48,865 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:49,582 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:50,961 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:15:53,587 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:54,478 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:15:57,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:15:57,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:15:57,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:15:57,592 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:15:59,645 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:01,597 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:03,600 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:05,236 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:07,605 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:10,412 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:11,610 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:12,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:16:12,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:16:12,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:16:15,615 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:15,994 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:16,583 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:16:16,584 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:16:16,585 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:16:16,587 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:16:16,617 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:16:17,618 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:20,963 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:16:21,623 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:21,964 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:25,628 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:27,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:16:27,083 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:27,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:16:27,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:16:29,633 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:32,946 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:33,638 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:35,641 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:38,141 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:39,646 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:42,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:16:42,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:16:42,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:16:42,649 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:43,223 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:46,676 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:48,324 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:50,681 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:50,965 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:16:52,719 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:16:52,720 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:16:52,721 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:16:52,723 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:16:53,685 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:16:53,723 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:16:54,687 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:56,689 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:16:57,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:16:57,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:16:57,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:16:59,286 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:00,694 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:04,699 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:04,749 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:08,705 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:10,417 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:12,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:17:12,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:17:12,094 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:17:12,710 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:15,702 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:16,715 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:18,718 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:20,967 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:17:20,968 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:22,723 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:26,732 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:26,777 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:27,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:17:27,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:17:27,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:17:29,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:17:29,411 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:17:29,411 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:17:29,413 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:17:29,737 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:17:30,738 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:32,414 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:34,743 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:37,666 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:38,748 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:40,751 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:42,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:17:42,094 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:17:42,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:17:43,251 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:44,758 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:48,527 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:48,763 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:50,982 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:17:52,768 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:54,149 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:17:56,773 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:57,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:17:57,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:17:57,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:17:58,776 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:17:59,365 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:02,781 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:04,980 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:05,584 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:18:05,586 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:18:05,586 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:18:05,589 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:18:05,785 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:18:06,787 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:10,199 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:10,792 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:12,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:18:12,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:18:12,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:18:14,797 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:15,813 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:16,800 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:20,805 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:20,970 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:18:20,971 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:24,810 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:26,679 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:27,059 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:18:27,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:18:27,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:18:28,815 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:31,897 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:32,820 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:34,823 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:37,508 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:38,828 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:41,721 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:18:41,722 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:18:41,723 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:18:41,725 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:18:41,851 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:18:42,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:18:42,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:18:42,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:18:42,853 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:43,217 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:46,955 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:48,333 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:50,960 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:50,972 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:18:52,963 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:53,556 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:18:56,969 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:18:57,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:18:57,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:18:57,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:18:59,215 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:00,974 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:04,390 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:04,979 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:08,984 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:09,992 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:10,986 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:12,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:19:12,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:19:12,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:19:14,991 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:15,212 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:17,824 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:19:17,825 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:19:17,826 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:19:17,827 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:19:17,996 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:19:18,997 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:20,828 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:20,980 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:19:23,003 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:26,038 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:27,009 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:27,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:19:27,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:19:27,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:19:29,012 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:31,662 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:33,016 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:36,865 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:37,021 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:41,026 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:42,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:19:42,084 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:42,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:19:42,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:19:45,031 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:47,034 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:47,673 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:50,992 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:19:51,039 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:52,994 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:53,868 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:19:53,869 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:19:53,870 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:19:53,871 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:19:54,043 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:19:55,045 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:19:57,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:19:57,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:19:57,095 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:19:58,478 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:19:59,050 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:03,055 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:04,086 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:07,059 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:09,062 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:09,310 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:12,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:20:12,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:20:12,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:20:13,067 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:15,220 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:17,072 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:20,987 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:20:20,988 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:21,077 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:25,082 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:26,345 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:27,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:20:27,085 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:27,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:20:27,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:20:29,960 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:20:29,961 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:20:29,961 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:20:29,963 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:20:30,089 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:20:31,091 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:31,964 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:35,095 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:37,170 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:39,100 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:42,060 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:20:42,095 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:20:42,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:20:42,222 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:43,105 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:45,114 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:47,363 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:49,119 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:50,986 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:20:52,562 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:20:53,124 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:57,061 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:20:57,096 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:20:57,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:20:57,129 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:20:58,250 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:01,134 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:03,136 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:03,376 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:05,987 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:21:05,988 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:21:05,989 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:21:05,992 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:21:06,141 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:21:07,143 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:08,993 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:11,147 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:12,061 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:21:12,096 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:21:12,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:21:14,194 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:15,152 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:19,157 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:19,803 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:20,988 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:21:21,160 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:25,012 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:25,164 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:27,061 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:21:27,096 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:21:27,096 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:21:29,169 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:30,617 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:33,174 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:35,817 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:37,179 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:39,182 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:41,416 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:42,019 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:21:42,020 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:21:42,020 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:21:42,023 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:21:42,141 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:21:42,141 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:21:42,142 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:21:42,186 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:21:43,187 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:46,619 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:47,192 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:50,990 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:21:51,197 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:51,990 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:55,203 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:21:57,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:21:57,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:21:57,105 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:21:57,117 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:21:57,205 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:01,210 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:03,020 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:05,215 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:08,215 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:09,220 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:12,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:22:12,105 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:22:12,117 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:22:13,225 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:14,200 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:15,228 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:18,003 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:22:18,004 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:22:18,004 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:22:18,007 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:22:18,233 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:22:19,234 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:20,008 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:20,991 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:22:23,239 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:25,190 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:27,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:22:27,106 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:22:27,117 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:22:27,338 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:30,787 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:31,343 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:33,346 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:35,984 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:37,351 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:41,356 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:41,585 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:42,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:22:42,105 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:22:42,117 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:22:45,361 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:46,784 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:49,365 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:50,993 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:22:51,368 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:51,994 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:53,988 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:22:53,989 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:22:53,989 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:22:53,991 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:22:54,372 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:22:55,373 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:22:57,105 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:22:57,106 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:22:57,106 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:22:57,117 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:22:59,379 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:02,184 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:03,383 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:07,388 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:07,792 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:09,391 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:12,134 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:23:12,135 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:23:12,135 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:23:13,279 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:13,396 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:17,401 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:18,614 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:21,007 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:23:21,406 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:23,821 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:25,411 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:27,121 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:23:27,133 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:23:27,133 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:23:29,198 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:29,416 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:30,409 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:23:30,410 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:23:30,410 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:23:30,412 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:23:30,418 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:23:31,419 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:35,016 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:35,424 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:39,429 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:40,618 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:42,121 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:23:42,133 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:23:42,133 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:23:43,434 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:45,810 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:47,439 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:49,441 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:50,997 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:23:50,998 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:51,444 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:53,447 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:55,449 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:56,315 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:23:57,121 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:23:57,133 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:23:57,133 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:23:57,452 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:23:59,455 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:01,458 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:01,806 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:03,460 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:05,463 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:06,868 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:07,466 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:09,468 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:11,471 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:11,868 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:12,121 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:24:12,133 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:24:12,133 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:24:13,474 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:15,476 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:17,369 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:17,479 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:18,932 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:24:18,933 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:24:18,934 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:24:18,936 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:24:19,482 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:19,483 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:24:20,999 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:24:21,485 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:23,000 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:25,491 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:27,290 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:24:27,291 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:24:27,291 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:24:28,504 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:29,497 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:29,893 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: partial_history +2024-02-01 19:24:29,898 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 19:24:29,898 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 19:24:29,898 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 19:24:29,898 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 19:24:29,899 DEBUG SenderThread:239784 [sender.py:send():382] send: metric +2024-02-01 19:24:29,899 DEBUG SenderThread:239784 [sender.py:send():382] send: history +2024-02-01 19:24:29,899 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: summary_record +2024-02-01 19:24:29,900 INFO SenderThread:239784 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-02-01 19:24:30,499 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/wandb-summary.json +2024-02-01 19:24:31,500 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/output.log +2024-02-01 19:24:33,901 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:38,902 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:42,251 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:24:42,251 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:24:42,251 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:24:44,406 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:49,407 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:51,001 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:24:55,002 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:24:57,251 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:24:57,251 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:24:57,252 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:25:00,352 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:25:00,535 INFO Thread-12 :239784 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft/wandb/run-20240201_175850-i93q0p12/files/config.yaml +2024-02-01 19:25:05,479 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:25:10,479 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:25:12,252 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:25:12,253 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:25:12,253 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status +2024-02-01 19:25:16,307 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:25:21,003 DEBUG SenderThread:239784 [sender.py:send():382] send: stats +2024-02-01 19:25:22,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:25:27,004 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: status_report +2024-02-01 19:25:27,251 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-01 19:25:27,252 DEBUG HandlerThread:239784 [handler.py:handle_request():146] handle_request: stop_status +2024-02-01 19:25:27,253 DEBUG SenderThread:239784 [sender.py:send_request():409] send_request: stop_status