diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,1089 @@ +2024-04-24 16:43:24,540 INFO StreamThr :1854781 [internal.py:wandb_internal():86] W&B internal server running at pid: 1854781, started at: 2024-04-24 16:43:24.539371 +2024-04-24 16:43:24,542 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status +2024-04-24 16:43:24,545 INFO WriterThread:1854781 [datastore.py:open_for_write():85] open: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/run-xfbnm7qo.wandb +2024-04-24 16:43:24,546 DEBUG SenderThread:1854781 [sender.py:send():382] send: header +2024-04-24 16:43:24,571 DEBUG SenderThread:1854781 [sender.py:send():382] send: run +2024-04-24 16:43:24,843 INFO SenderThread:1854781 [dir_watcher.py:__init__():211] watching files in: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files +2024-04-24 16:43:24,843 INFO SenderThread:1854781 [sender.py:_start_run_threads():1136] run started: xfbnm7qo with start time 1713977004.542006 +2024-04-24 16:43:24,850 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: check_version +2024-04-24 16:43:24,850 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: check_version +2024-04-24 16:43:24,946 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: run_start +2024-04-24 16:43:25,004 DEBUG HandlerThread:1854781 [system_info.py:__init__():32] System info init +2024-04-24 16:43:25,004 DEBUG HandlerThread:1854781 [system_info.py:__init__():47] System info init done +2024-04-24 16:43:25,004 INFO HandlerThread:1854781 [system_monitor.py:start():194] Starting system monitor +2024-04-24 16:43:25,004 INFO SystemMonitor:1854781 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-24 16:43:25,004 INFO HandlerThread:1854781 [system_monitor.py:probe():214] Collecting system info +2024-04-24 16:43:25,005 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started cpu monitoring +2024-04-24 16:43:25,005 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started disk monitoring +2024-04-24 16:43:25,006 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started gpu monitoring +2024-04-24 16:43:25,006 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started memory monitoring +2024-04-24 16:43:25,007 INFO SystemMonitor:1854781 [interfaces.py:start():190] Started network monitoring +2024-04-24 16:43:25,057 DEBUG HandlerThread:1854781 [system_info.py:probe():196] Probing system +2024-04-24 16:43:25,060 DEBUG HandlerThread:1854781 [system_info.py:_probe_git():181] Probing git +2024-04-24 16:43:25,080 DEBUG HandlerThread:1854781 [system_info.py:_probe_git():189] Probing git done +2024-04-24 16:43:25,080 DEBUG HandlerThread:1854781 [system_info.py:probe():244] Probing system done +2024-04-24 16:43:25,080 DEBUG HandlerThread:1854781 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1048-aws-x86_64-with-glibc2.31', 'python': '3.11.5', 'heartbeatAt': '2024-04-24T16:43:25.058035', 'startedAt': '2024-04-24T16:43:24.523748', 'docker': None, 'cuda': None, 'args': ('config_full.yaml',), 'state': 'running', 'program': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/run_sft.py', 'codePathLocal': 'run_sft.py', 'codePath': 'run_sft.py', 'git': {'remote': 'https://huggingface.co/sanchit-gandhi/distil-zephyr-1.5b-ssft-ultrachat', 'commit': 'cbea69c6b95c970317a1e47c3f614b55b33f8ed9'}, 'email': None, 'root': '/fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat', 'host': 'ip-26-0-162-233', 'username': 'sanchit', 'executable': '/fsx/sanchit/miniconda3/envs/venv/bin/python', 'cpu_count': 96, 'cpu_count_logical': 96, 'cpu_freq': {'current': 2729.8387291666663, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.161, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3584.12, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.175, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.329, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3596.81, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.102, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3596.611, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 3598.198, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}, {'current': 2650.0, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 290.7472343444824, 'used': 59.25613021850586}}, 'gpu': 'NVIDIA H100 80GB HBM3', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}, {'name': 'NVIDIA H100 80GB HBM3', 'memory_total': 85520809984}], 'memory': {'total': 1999.9855270385742}} +2024-04-24 16:43:25,081 INFO HandlerThread:1854781 [system_monitor.py:probe():224] Finished collecting system info +2024-04-24 16:43:25,081 INFO HandlerThread:1854781 [system_monitor.py:probe():227] Publishing system info +2024-04-24 16:43:25,081 DEBUG HandlerThread:1854781 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2024-04-24 16:43:25,082 DEBUG HandlerThread:1854781 [system_info.py:_save_pip():68] Saving pip packages done +2024-04-24 16:43:25,083 DEBUG HandlerThread:1854781 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2024-04-24 16:43:25,845 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/conda-environment.yaml +2024-04-24 16:43:25,845 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/requirements.txt +2024-04-24 16:43:30,500 DEBUG HandlerThread:1854781 [system_info.py:_save_conda():87] Saving conda packages done +2024-04-24 16:43:30,502 INFO HandlerThread:1854781 [system_monitor.py:probe():229] Finished publishing system info +2024-04-24 16:43:30,525 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:43:30,525 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: keepalive +2024-04-24 16:43:30,525 DEBUG SenderThread:1854781 [sender.py:send():382] send: files +2024-04-24 16:43:30,526 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now +2024-04-24 16:43:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:43:30,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:43:30,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:43:30,621 DEBUG SenderThread:1854781 [sender.py:send():382] send: telemetry +2024-04-24 16:43:30,622 DEBUG SenderThread:1854781 [sender.py:send():382] send: config +2024-04-24 16:43:30,622 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric +2024-04-24 16:43:30,623 DEBUG SenderThread:1854781 [sender.py:send():382] send: telemetry +2024-04-24 16:43:30,623 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric +2024-04-24 16:43:30,623 WARNING SenderThread:1854781 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2024-04-24 16:43:30,623 DEBUG SenderThread:1854781 [sender.py:send():382] send: telemetry +2024-04-24 16:43:30,795 INFO wandb-upload_0:1854781 [upload_job.py:push():131] Uploaded file /tmp/tmp2ljkzqevwandb/om53ihff-wandb-metadata.json +2024-04-24 16:43:30,849 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/conda-environment.yaml +2024-04-24 16:43:30,849 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-metadata.json +2024-04-24 16:43:30,849 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:32,851 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:33,571 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:43:33,573 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric +2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric +2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric +2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: metric +2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:43:33,574 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:43:33,576 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:43:33,852 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_created():271] file/dir created: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:43:34,854 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:35,578 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:43:35,855 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:36,855 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:38,858 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:40,807 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:43:40,860 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:42,861 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:44,863 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:43:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:43:45,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:43:46,660 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:43:46,865 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:48,867 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:50,869 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:52,488 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:43:52,871 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:54,873 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:56,875 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:43:58,187 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:43:58,877 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/config.yaml +2024-04-24 16:43:58,878 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:00,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:44:00,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:44:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:44:00,879 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:02,881 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:03,895 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:04,883 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:06,885 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:08,887 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:09,580 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:10,889 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:12,891 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:14,893 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:15,263 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:15,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:44:15,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:44:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:44:16,895 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:18,897 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:19,965 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:44:19,967 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:44:19,967 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:44:19,969 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:44:20,899 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:44:20,899 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:20,970 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:21,900 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:22,901 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:24,903 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:25,007 DEBUG SystemMonitor:1854781 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-24 16:44:25,011 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:44:26,624 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:26,905 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:28,907 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:44:30,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:44:30,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:44:30,908 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:32,318 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:32,910 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:34,912 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:36,914 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:38,005 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:38,916 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:40,918 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:42,920 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:43,687 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:44,922 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:44:45,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:44:45,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:44:46,924 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:48,925 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:49,361 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:50,927 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:52,929 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:54,931 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:55,012 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:44:55,013 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:44:56,933 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:44:58,935 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:00,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:45:00,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:45:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:45:00,698 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:00,937 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:02,939 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:04,941 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:06,452 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:06,942 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:07,375 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:45:07,377 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:45:07,377 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:45:07,379 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:45:07,944 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:45:08,945 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:09,946 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:10,947 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:12,142 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:12,949 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:14,951 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:15,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:45:15,532 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:45:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:45:16,953 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:17,819 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:18,955 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:20,957 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:22,958 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:23,488 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:24,960 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:25,023 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:45:26,962 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:28,964 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:29,122 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:45:30,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:45:30,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:45:30,966 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:32,968 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:34,739 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:34,970 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:36,972 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:38,973 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:40,343 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:40,975 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:42,977 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:44,979 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:45,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:45:45,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:45:45,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:45:45,667 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:46,981 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:48,983 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:50,985 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:51,602 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:52,987 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:54,402 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:45:54,404 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:45:54,404 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:45:54,405 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:45:54,989 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:45:54,990 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:55,016 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:45:55,990 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:56,991 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:45:57,250 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:45:58,993 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:00,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:46:00,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:46:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:46:00,995 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:02,892 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:02,997 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:04,999 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:07,000 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:08,529 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:09,002 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:11,004 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:13,006 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:14,157 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:15,008 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:15,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:46:15,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:46:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:46:17,010 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:19,012 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:19,805 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:21,014 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:23,016 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:25,018 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:46:25,019 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:25,019 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:27,021 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:29,023 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:30,090 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:30,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:46:30,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:46:30,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:46:31,025 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:33,027 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:35,029 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:35,728 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:37,031 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:39,033 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:41,035 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:41,336 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:41,362 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:46:41,364 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:46:41,364 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:46:41,366 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:46:42,036 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:46:43,037 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:44,038 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:45,039 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:45,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:46:45,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:46:45,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:46:46,613 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:47,041 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:49,043 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:51,045 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:51,657 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:53,047 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:55,020 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:46:55,049 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:57,051 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:46:57,256 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:46:59,052 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:00,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:47:00,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:47:00,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:47:01,054 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:02,864 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:03,056 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:05,058 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:07,060 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:08,484 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:09,062 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:11,064 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:13,066 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:14,088 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:15,068 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:15,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:47:15,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:47:15,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:47:17,070 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:19,072 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:19,691 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:21,073 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:23,075 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:25,022 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:47:25,023 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:25,077 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:27,079 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:28,045 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:47:28,047 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:47:28,047 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:47:28,049 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:47:28,081 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:47:29,082 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:30,083 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:30,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:47:30,533 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:47:30,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:47:30,699 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:31,084 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:33,085 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:35,087 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:36,463 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:37,089 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:39,091 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:41,093 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:42,044 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:43,095 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:45,097 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:45,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:47:45,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:47:45,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:47:47,099 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:47,612 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:49,101 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:51,102 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:53,104 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:53,199 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:55,024 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:47:55,106 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:57,108 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:47:58,737 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:47:59,110 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:00,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:48:00,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:48:00,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:48:01,112 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:03,114 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:04,295 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:05,115 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:07,118 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:09,120 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:09,834 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:11,121 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:13,123 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:14,418 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:48:14,419 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:48:14,420 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:48:14,422 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:48:15,126 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:48:15,126 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:15,422 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:15,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:48:15,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:48:15,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:48:16,127 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:17,128 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:19,130 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:20,943 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:21,131 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:23,133 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:25,026 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:48:25,135 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:26,510 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:27,137 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:29,139 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:30,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:48:30,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:48:30,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:48:31,141 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:32,087 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:33,143 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:35,145 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:37,147 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:37,670 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:39,149 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:41,151 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:43,153 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:43,245 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:45,154 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:45,533 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:48:45,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:48:45,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:48:47,156 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:48,827 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:49,158 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:51,161 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:53,162 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:54,384 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:48:55,040 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:48:55,164 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:57,166 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:59,168 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:48:59,926 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:00,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:49:00,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:49:00,535 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:49:00,822 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:49:00,823 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:49:00,823 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:49:00,825 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:49:01,170 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:49:01,171 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:02,171 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:03,172 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:05,174 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:05,484 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:07,176 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:09,178 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:11,052 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:11,180 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:13,182 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:15,184 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:15,534 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:49:15,534 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:49:15,536 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:49:16,624 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:17,186 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:19,188 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:21,190 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:22,191 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:23,192 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:25,030 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:49:25,194 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:27,196 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:27,746 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:29,198 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:30,570 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:49:30,571 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:49:30,571 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:49:31,200 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:33,202 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:33,300 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:35,203 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:37,205 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:38,828 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:39,207 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:41,209 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:43,211 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:44,336 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:45,213 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:45,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:49:45,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:49:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:49:47,058 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:49:47,059 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:49:47,060 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:49:47,061 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:49:47,215 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:49:47,215 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:48,216 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:49,217 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:49,858 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:51,219 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:53,221 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:55,032 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:49:55,033 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:49:55,223 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:57,225 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:49:59,227 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:00,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:50:00,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:50:00,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:50:00,701 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:01,229 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:03,231 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:05,233 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:06,467 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:07,235 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:09,236 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:11,238 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:12,009 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:13,240 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:15,242 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:15,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:50:15,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:50:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:50:17,244 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:17,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:19,246 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:21,248 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:23,102 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:23,250 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:25,034 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:50:25,252 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:27,254 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:28,648 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:29,256 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:30,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:50:30,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:50:30,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:50:31,258 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:33,218 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:50:33,219 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:50:33,220 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:50:33,221 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:50:33,260 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:50:33,261 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:34,222 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:35,262 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:36,263 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:37,264 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:39,266 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:39,743 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:41,268 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:43,270 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:45,270 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:45,272 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:45,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:50:45,552 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:50:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:50:47,274 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:49,276 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:50,784 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:51,278 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:53,280 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:55,072 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:50:55,281 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:56,317 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:50:57,283 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:50:59,285 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:00,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:51:00,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:51:00,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:51:01,287 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:01,840 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:03,289 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:05,291 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:07,293 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:07,360 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:09,295 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:11,296 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:12,884 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:13,299 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:15,300 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:15,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:51:15,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:51:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:51:17,302 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:18,392 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:19,268 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:51:19,269 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:51:19,270 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:51:19,271 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:51:19,305 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:51:19,305 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:21,307 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:22,308 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:23,309 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:23,942 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:25,038 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:51:25,311 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:27,312 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:29,314 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:29,487 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:30,552 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:51:30,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:51:30,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:51:31,317 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:33,319 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:34,977 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:35,320 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:37,322 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:39,324 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:40,471 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:41,326 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:43,328 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:45,330 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:45,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:51:45,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:51:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:51:45,662 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:47,331 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:49,333 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:51,335 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:51,427 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:53,337 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:55,040 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:51:55,339 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:56,920 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:51:57,341 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:51:59,343 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:00,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:52:00,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:52:00,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:52:01,346 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:02,415 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:03,348 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:05,117 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:52:05,118 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:52:05,118 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:52:05,120 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:52:05,351 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:52:05,351 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:06,352 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:07,354 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:07,925 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:09,358 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:11,364 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:13,367 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:13,429 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:15,371 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:15,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:52:15,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:52:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:52:17,375 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:18,951 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:19,380 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:21,385 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:23,389 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:24,470 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:25,042 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:52:25,393 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:27,398 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:29,409 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:29,954 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:30,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:52:30,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:52:30,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:52:31,413 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:33,418 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:35,424 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:35,435 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:37,426 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:39,428 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:40,923 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:41,431 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:43,433 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:45,435 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:45,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:52:45,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:52:45,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:52:46,692 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:47,437 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:49,439 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:50,949 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:52:50,951 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:52:50,951 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:52:50,953 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:52:51,442 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:52:51,442 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:51,953 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:52,443 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:53,446 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:55,044 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:52:55,448 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:57,431 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:52:57,450 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:52:59,452 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:00,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:53:00,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:53:00,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:53:01,453 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:02,948 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:03,455 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:05,457 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:07,459 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:08,459 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:09,461 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:11,463 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:13,465 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:13,977 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:15,467 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:15,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:53:15,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:53:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:53:17,469 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:19,472 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:19,488 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:21,685 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:23,688 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:25,005 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:25,046 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:53:25,691 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:27,698 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:29,702 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:30,532 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:30,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:53:30,553 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:53:30,568 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:53:31,704 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:33,706 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:35,708 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:36,033 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:36,891 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:53:36,894 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:53:36,894 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:53:36,905 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:53:37,710 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:53:37,711 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:39,712 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:41,547 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:41,714 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:43,716 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:45,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:53:45,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:53:45,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:53:45,718 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:47,029 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:47,720 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:49,722 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:51,724 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:52,531 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:53,726 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:55,048 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:53:55,728 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:57,730 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:53:58,028 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:53:59,732 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:00,553 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:54:00,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:54:00,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:54:01,734 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:03,525 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:03,736 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:05,737 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:07,739 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:09,010 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:09,741 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:11,743 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:13,745 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:14,495 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:15,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:54:15,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:54:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:54:15,747 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:17,749 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:19,751 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:19,951 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:21,753 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:22,601 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:54:22,603 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:54:22,603 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:54:22,605 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:54:22,755 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:54:23,755 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:25,060 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:54:25,060 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:25,757 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:27,759 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:29,761 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:30,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:54:30,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:54:30,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:54:30,625 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:31,763 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:33,765 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:35,767 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:36,349 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:37,769 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:39,770 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:41,772 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:41,837 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:43,774 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:45,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:54:45,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:54:45,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:54:45,776 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:47,336 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:47,778 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:49,780 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:51,781 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:52,840 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:53,783 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:55,053 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:54:55,785 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:57,787 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:54:58,343 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:54:59,789 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:00,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:55:00,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:55:00,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:55:01,791 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:03,792 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:03,853 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:05,794 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:07,796 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:08,387 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:55:08,388 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:55:08,389 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:55:08,390 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:55:08,798 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:55:09,391 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:09,799 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:11,801 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:13,802 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:14,878 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:15,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:55:15,554 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:55:15,565 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:55:15,804 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:17,806 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:19,808 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:20,367 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:21,810 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:23,812 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:25,055 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:55:25,813 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:26,055 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:27,815 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:29,817 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:30,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:55:30,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:55:30,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:55:31,656 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:31,819 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:33,821 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:35,823 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:36,734 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:37,824 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:39,826 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:41,828 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:42,203 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:43,830 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:45,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:55:45,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:55:45,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:55:45,832 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:47,668 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:47,834 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:49,836 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:51,838 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:53,139 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:53,839 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:54,003 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:55:54,005 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:55:54,005 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:55:54,007 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:55:54,841 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:55:55,056 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:55:55,842 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:57,844 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:55:58,637 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:55:59,846 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:00,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:56:00,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:56:00,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:56:01,848 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:03,850 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:04,136 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:05,852 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:07,854 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:09,643 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:09,855 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:11,857 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:13,859 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:15,120 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:15,554 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:56:15,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:56:15,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:56:15,861 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:17,863 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:19,865 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:20,612 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:21,867 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:23,869 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:25,068 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:56:25,871 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:26,117 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:27,873 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:29,875 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:30,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:56:30,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:56:30,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:56:31,717 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:31,877 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:33,879 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:35,881 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:37,074 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:37,883 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:39,746 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:56:39,748 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:56:39,748 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:56:39,750 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:56:39,885 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:56:39,886 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:41,887 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:42,546 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:43,889 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:45,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:56:45,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:56:45,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:56:45,891 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:47,893 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:48,021 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:49,895 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:51,897 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:53,490 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:53,898 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:55,060 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:56:55,900 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:57,902 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:56:58,986 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:56:59,904 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:00,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:57:00,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:57:00,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:57:01,906 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:03,908 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:04,466 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:05,910 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:07,912 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:09,914 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:09,952 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:11,916 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:13,917 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:15,446 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:15,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:57:15,555 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:57:15,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:57:15,919 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:17,921 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:19,923 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:20,944 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:21,925 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:23,927 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:25,062 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:57:25,467 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:57:25,469 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:57:25,469 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:57:25,470 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:57:25,929 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:57:25,930 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:26,471 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:27,931 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:29,933 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:30,555 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:57:30,556 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:57:30,566 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:57:30,934 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:31,912 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:32,936 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:34,938 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:36,940 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:37,373 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:38,942 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:40,944 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:42,825 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:42,945 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:44,947 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:45,576 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:57:45,576 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:57:45,576 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:57:46,949 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:48,286 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:48,951 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:50,953 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:52,955 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:53,746 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:57:54,956 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:55,064 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:57:56,958 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:58,960 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:57:59,216 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:00,563 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:58:00,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:58:00,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:58:00,962 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:02,964 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:04,703 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:04,966 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:06,968 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:08,970 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:10,203 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:10,972 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:11,053 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:58:11,055 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:58:11,055 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:58:11,057 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:58:11,974 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:58:11,974 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:12,975 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:14,977 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:58:15,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:15,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:58:15,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:58:16,978 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:18,980 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:20,982 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:21,213 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:22,984 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:24,986 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:25,066 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:58:26,714 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:26,988 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:28,990 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:30,563 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:58:30,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:58:30,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:58:30,992 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:32,197 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:32,994 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:34,996 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:36,999 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:37,677 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:39,001 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:41,002 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:43,004 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:43,163 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:45,006 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:45,564 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:58:45,564 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:58:45,569 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:58:47,009 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:48,630 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:49,011 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:51,013 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:53,015 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:54,110 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:58:55,016 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:55,070 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:58:56,778 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:58:56,779 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:58:56,779 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:58:56,782 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:58:57,019 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:58:57,019 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:58,020 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:58:59,789 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:01,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:59:01,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:59:01,069 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:59:03,025 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:05,171 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:10,171 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:13,035 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:15,037 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:16,015 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:16,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:59:16,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:59:16,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:59:17,038 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:19,040 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:21,042 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:21,480 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:23,044 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:25,046 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:25,072 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:59:26,944 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:27,048 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:29,049 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:31,051 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:31,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:59:31,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:59:31,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:59:32,434 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:33,053 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:35,055 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:37,057 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:37,910 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:39,059 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:41,060 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:43,062 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:43,396 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:45,064 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:46,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 16:59:46,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 16:59:46,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 16:59:47,066 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:48,885 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:49,068 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:51,070 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:53,071 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:54,358 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 16:59:55,074 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:55,083 DEBUG SenderThread:1854781 [sender.py:send():382] send: stats +2024-04-24 16:59:57,039 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: partial_history +2024-04-24 16:59:57,040 DEBUG SenderThread:1854781 [sender.py:send():382] send: history +2024-04-24 16:59:57,040 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: summary_record +2024-04-24 16:59:57,042 INFO SenderThread:1854781 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2024-04-24 16:59:57,077 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/wandb-summary.json +2024-04-24 16:59:57,077 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:58,078 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:59,079 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 16:59:59,841 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 17:00:01,067 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-24 17:00:01,068 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: stop_status +2024-04-24 17:00:01,068 DEBUG SenderThread:1854781 [sender.py:send_request():409] send_request: stop_status +2024-04-24 17:00:01,081 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 17:00:03,082 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 17:00:05,084 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 17:00:05,331 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 17:00:07,086 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 17:00:09,088 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log +2024-04-24 17:00:10,811 DEBUG HandlerThread:1854781 [handler.py:handle_request():146] handle_request: status_report +2024-04-24 17:00:11,090 INFO Thread-12 :1854781 [dir_watcher.py:_on_file_modified():288] file/dir modified: /fsx/sanchit/distil-zephyr-1.5b-ssft-ultrachat/wandb/run-20240424_164324-xfbnm7qo/files/output.log