diff --git "a/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/logs/debug-internal.log" "b/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/logs/debug-internal.log"
new file mode 100644--- /dev/null
+++ "b/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/logs/debug-internal.log"
@@ -0,0 +1,6944 @@
+2023-11-17 08:56:56,515 INFO    StreamThr :11593 [internal.py:wandb_internal():86] W&B internal server running at pid: 11593, started at: 2023-11-17 08:56:56.514971
+2023-11-17 08:56:56,519 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status
+2023-11-17 08:56:56,530 INFO    WriterThread:11593 [datastore.py:open_for_write():85] open: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/run-hf8ykeyx.wandb
+2023-11-17 08:56:56,530 DEBUG   SenderThread:11593 [sender.py:send():379] send: header
+2023-11-17 08:56:56,537 DEBUG   SenderThread:11593 [sender.py:send():379] send: run
+2023-11-17 08:56:56,907 INFO    SenderThread:11593 [dir_watcher.py:__init__():211] watching files in: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files
+2023-11-17 08:56:56,908 INFO    SenderThread:11593 [sender.py:_start_run_threads():1122] run started: hf8ykeyx with start time 1700211416.520984
+2023-11-17 08:56:56,908 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:56:56,908 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:56:56,911 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: check_version
+2023-11-17 08:56:56,911 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: check_version
+2023-11-17 08:56:57,193 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: run_start
+2023-11-17 08:56:57,202 DEBUG   HandlerThread:11593 [system_info.py:__init__():31] System info init
+2023-11-17 08:56:57,202 DEBUG   HandlerThread:11593 [system_info.py:__init__():46] System info init done
+2023-11-17 08:56:57,203 INFO    HandlerThread:11593 [system_monitor.py:start():181] Starting system monitor
+2023-11-17 08:56:57,203 INFO    SystemMonitor:11593 [system_monitor.py:_start():145] Starting system asset monitoring threads
+2023-11-17 08:56:57,203 INFO    HandlerThread:11593 [system_monitor.py:probe():201] Collecting system info
+2023-11-17 08:56:57,204 INFO    SystemMonitor:11593 [interfaces.py:start():190] Started cpu monitoring
+2023-11-17 08:56:57,205 INFO    SystemMonitor:11593 [interfaces.py:start():190] Started disk monitoring
+2023-11-17 08:56:57,207 INFO    SystemMonitor:11593 [interfaces.py:start():190] Started gpu monitoring
+2023-11-17 08:56:57,207 INFO    SystemMonitor:11593 [interfaces.py:start():190] Started memory monitoring
+2023-11-17 08:56:57,208 INFO    SystemMonitor:11593 [interfaces.py:start():190] Started network monitoring
+2023-11-17 08:56:57,211 DEBUG   HandlerThread:11593 [system_info.py:probe():195] Probing system
+2023-11-17 08:56:57,219 DEBUG   HandlerThread:11593 [system_info.py:_probe_git():180] Probing git
+2023-11-17 08:56:57,236 DEBUG   HandlerThread:11593 [system_info.py:_probe_git():188] Probing git done
+2023-11-17 08:56:57,236 DEBUG   HandlerThread:11593 [system_info.py:probe():240] Probing system done
+2023-11-17 08:56:57,237 DEBUG   HandlerThread:11593 [system_monitor.py:probe():210] {'os': 'Linux-5.15.120+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2023-11-17T08:56:57.212063', 'startedAt': '2023-11-17T08:56:56.507822', 'docker': None, 'cuda': None, 'args': ('--sample_prompts=/content/LoRA/config/sample_prompt.toml', '--config_file=/content/LoRA/config/config_file.toml', '--wandb_api_key=540d5e6b5446a44f2f5c8269e0da63b4c7ad408c'), 'state': 'running', 'program': '/content/kohya-trainer/sdxl_train_network.py', 'codePath': 'sdxl_train_network.py', 'git': {'remote': 'https://github.com/qaneel/kohya-trainer', 'commit': '05811296f6dc987f67f194689e106a326017b9d4'}, 'email': None, 'root': '/content/kohya-trainer', 'host': '3af72b0d0270', 'username': 'root', 'executable': '/usr/bin/python3', 'cpu_count': 1, 'cpu_count_logical': 2, 'cpu_freq': {'current': 2299.998, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2299.998, 'min': 0.0, 'max': 0.0}, {'current': 2299.998, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 78.1898422241211, 'used': 41.9006462097168}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 12.6783447265625}}
+2023-11-17 08:56:57,237 INFO    HandlerThread:11593 [system_monitor.py:probe():211] Finished collecting system info
+2023-11-17 08:56:57,237 INFO    HandlerThread:11593 [system_monitor.py:probe():214] Publishing system info
+2023-11-17 08:56:57,237 DEBUG   HandlerThread:11593 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
+2023-11-17 08:56:57,238 DEBUG   HandlerThread:11593 [system_info.py:_save_pip():67] Saving pip packages done
+2023-11-17 08:56:57,238 INFO    HandlerThread:11593 [system_monitor.py:probe():216] Finished publishing system info
+2023-11-17 08:56:57,252 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 08:56:57,252 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-metadata.json with policy now
+2023-11-17 08:56:57,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:56:57,254 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:56:57,343 DEBUG   SenderThread:11593 [sender.py:send():379] send: telemetry
+2023-11-17 08:56:57,909 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 08:56:57,910 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/requirements.txt
+2023-11-17 08:56:57,910 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:56:57,910 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-metadata.json
+2023-11-17 08:56:57,990 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/rb5exyyf-wandb-metadata.json
+2023-11-17 08:56:59,909 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 08:57:02,352 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:07,353 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:12,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:57:12,255 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:57:12,384 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:13,089 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:17,385 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:19,363 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:19,365 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:19,365 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:19,370 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:19,917 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:57:23,371 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:26,744 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:26,745 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:26,745 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:26,747 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:26,920 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:57:27,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:57:27,255 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:57:29,312 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:29,921 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/config.yaml
+2023-11-17 08:57:34,357 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:34,766 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:34,767 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:34,768 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:34,768 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:34,923 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:57:39,769 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:41,130 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:41,132 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:41,132 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:41,134 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:41,926 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:57:42,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:57:42,255 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:57:45,320 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:47,788 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:47,790 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:47,790 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:47,792 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:47,929 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:57:50,793 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:52,354 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:52,355 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:52,355 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:52,357 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:52,931 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:57:56,359 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:57:57,208 DEBUG   SystemMonitor:11593 [system_monitor.py:_start():159] Starting system metrics aggregation loop
+2023-11-17 08:57:57,210 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 08:57:57,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:57:57,255 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:57:59,385 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:57:59,387 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:57:59,387 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:57:59,388 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:57:59,933 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:01,389 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:05,543 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:05,544 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:05,544 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:05,547 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:05,938 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:06,547 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:11,548 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:11,691 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:11,692 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:11,692 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:11,693 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:11,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:12,259 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:58:12,261 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:58:17,360 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:18,646 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:18,647 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:18,648 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:18,651 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:18,944 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:22,653 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:25,309 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:25,310 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:25,311 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:25,311 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:25,947 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:27,211 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 08:58:27,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:27,691 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:58:27,691 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:58:32,756 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:33,930 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:33,931 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:33,931 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:33,934 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:33,950 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:37,935 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:39,909 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:39,910 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:39,911 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:39,911 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:39,952 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:42,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:58:42,686 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:58:43,788 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:46,496 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:46,497 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:46,497 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:46,500 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:46,955 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:49,501 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:54,034 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:58:54,035 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:58:54,035 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:58:54,036 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:58:54,958 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:58:55,036 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:58:57,212 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 08:58:57,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:58:57,687 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:59:00,585 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:00,586 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:00,586 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:00,587 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:00,587 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:00,960 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:05,588 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:08,397 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:08,398 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:08,398 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:08,401 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:08,963 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:11,402 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:12,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:59:12,687 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:59:15,861 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:15,862 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:15,862 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:15,863 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:15,967 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:16,864 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:21,865 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:22,228 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:22,230 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:22,230 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:22,232 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:22,970 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:27,213 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 08:59:27,214 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:27,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:59:27,687 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:59:29,369 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:29,370 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:29,370 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:29,371 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:29,972 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:32,372 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:36,049 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:36,051 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:36,051 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:36,053 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:36,975 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:38,054 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:41,952 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:41,953 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:41,954 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:41,954 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:41,977 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:42,687 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:59:42,687 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 08:59:43,830 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:48,261 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:48,262 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:48,263 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:48,266 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:48,980 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:49,267 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:54,268 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 08:59:55,657 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 08:59:55,658 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 08:59:55,658 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 08:59:55,659 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 08:59:55,983 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 08:59:57,214 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 08:59:58,056 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 08:59:58,057 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:00:00,128 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:04,071 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:04,072 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:04,072 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:04,074 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:04,986 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:05,925 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:05,926 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:05,927 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:05,927 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:05,928 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:05,987 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:10,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:13,056 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:00:13,057 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:00:14,071 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:14,072 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:14,072 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:14,073 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:14,990 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:16,073 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:20,807 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:20,809 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:20,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:20,811 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:20,992 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:21,812 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:26,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:27,214 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:00:28,056 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:00:28,057 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:00:28,419 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:28,420 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:28,420 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:28,421 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:28,995 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:32,432 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:36,520 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:36,521 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:36,521 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:36,523 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:36,998 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:37,524 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:39,639 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:39,640 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:39,640 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:39,641 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:40,000 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:42,642 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:43,056 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:00:43,057 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:00:46,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:46,020 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:46,021 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:46,021 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:47,002 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:48,022 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:52,993 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:52,994 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:52,994 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:52,997 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:00:53,005 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:00:53,997 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:57,215 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:00:58,057 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:00:58,060 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:00:59,168 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:00:59,711 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:00:59,712 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:00:59,713 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:00:59,713 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:00,007 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:04,714 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:06,318 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:06,319 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:06,320 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:06,322 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:07,010 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:10,323 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:11,420 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:11,421 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:11,422 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:11,422 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:12,012 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:13,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:01:13,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:01:16,229 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:17,930 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:17,931 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:17,931 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:17,932 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:18,015 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:21,933 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:24,671 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:24,672 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:24,673 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:24,673 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:25,017 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:27,216 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:01:27,217 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:28,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:01:28,182 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:01:32,184 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:32,185 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:32,186 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:32,188 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:33,021 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:33,189 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:38,190 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:40,304 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:40,306 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:40,306 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:40,306 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:41,024 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:43,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:01:43,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:01:43,270 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:47,715 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:47,716 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:47,717 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:47,719 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:48,027 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:48,720 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:53,721 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:54,123 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:54,124 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:54,125 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:54,125 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:55,029 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:01:57,217 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:01:58,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:01:58,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:01:58,730 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:01:58,732 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:01:58,732 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:01:58,733 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:01:58,734 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:01:59,031 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:03,734 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:06,117 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:06,118 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:06,118 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:06,119 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:07,034 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:09,120 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:12,834 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:12,835 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:12,836 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:12,837 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:13,036 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:13,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:02:13,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:02:14,361 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:19,176 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:19,177 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:19,177 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:19,180 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:20,039 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:20,180 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:25,181 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:25,677 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:25,678 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:25,679 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:25,679 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:26,041 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:27,218 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:02:28,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:02:28,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:02:30,351 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:32,396 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:32,397 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:32,398 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:32,398 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:33,044 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:35,399 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:39,458 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:39,460 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:39,460 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:39,461 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:40,047 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:40,461 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:43,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:02:43,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:02:45,989 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:45,990 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:45,991 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:45,991 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:45,991 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:46,049 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:50,993 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:52,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:52,755 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:52,756 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:52,758 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:02:53,052 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:02:56,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:02:57,219 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:02:58,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:02:58,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:02:59,175 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:02:59,176 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:02:59,176 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:02:59,177 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:00,054 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:02,178 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:05,568 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:05,570 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:05,570 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:05,570 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:06,059 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:07,571 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:12,330 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:12,331 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:12,332 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:12,332 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:13,062 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:13,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:03:13,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:03:13,288 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:18,289 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:19,397 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:19,398 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:19,399 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:19,399 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:20,065 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:23,400 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:25,763 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:25,764 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:25,764 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:25,767 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:26,067 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:27,219 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:03:28,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:03:28,184 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:03:29,280 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:32,415 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:32,417 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:32,417 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:32,417 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:33,070 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:34,418 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:39,155 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:39,156 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:39,156 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:39,157 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:40,072 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:40,158 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:43,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:03:43,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:03:45,284 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:45,532 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:45,533 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:45,533 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:45,534 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:46,075 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:50,535 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:52,693 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:52,695 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:52,695 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:52,695 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:03:53,077 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:03:55,696 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:03:57,220 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:03:58,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:03:58,184 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:03:59,471 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:03:59,472 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:03:59,472 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:03:59,473 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:00,080 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:01,474 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:05,990 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:05,992 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:05,992 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:05,992 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:06,083 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:06,993 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:11,994 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:12,734 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:12,735 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:12,736 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:12,736 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:13,085 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:13,183 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:04:13,184 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:04:17,402 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:20,161 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:20,162 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:20,163 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:20,163 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:21,089 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:23,164 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:27,212 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:27,213 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:27,213 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:27,214 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:27,221 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:04:28,091 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:28,184 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:04:28,184 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:04:28,350 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:33,351 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:34,262 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:34,263 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:34,264 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:34,264 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:35,094 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:39,267 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:41,663 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:41,665 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:41,665 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:41,665 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:42,097 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:43,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:04:43,464 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:04:44,497 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:48,252 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:48,253 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:48,253 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:48,254 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:49,099 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:50,255 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:55,256 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:04:55,294 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:04:55,295 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:04:55,295 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:04:55,295 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:04:56,102 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:04:57,222 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:04:58,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:04:58,464 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:05:00,533 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:01,815 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:01,817 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:01,818 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:01,818 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:02,104 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:05,819 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:09,241 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:09,243 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:09,243 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:09,243 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:10,107 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:11,244 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:12,078 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:12,079 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:12,079 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:12,080 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:12,108 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:13,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:05:13,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:05:16,504 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:20,096 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:20,097 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:20,098 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:20,098 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:20,111 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:22,099 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:27,100 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:27,222 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:05:27,492 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:27,493 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:27,494 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:27,494 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:28,114 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:28,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:05:28,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:05:32,547 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:33,422 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:33,423 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:33,424 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:33,424 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:34,116 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:35,974 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:35,975 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:35,975 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:35,976 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:36,120 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:37,977 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:42,526 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:42,527 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:42,528 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:42,528 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:43,123 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:43,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:05:43,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:05:43,569 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:47,273 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:47,274 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:47,274 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:47,275 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:48,125 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:49,276 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:54,277 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:05:55,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:05:55,255 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:05:55,256 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:05:55,256 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:05:56,128 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:05:57,223 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:05:58,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:05:58,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:05:59,591 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:03,284 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:03,285 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:03,286 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:03,287 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:04,131 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:05,287 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:10,033 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:10,035 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:10,035 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:10,035 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:10,133 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:11,036 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:13,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:06:13,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:06:16,404 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:16,405 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:16,406 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:16,406 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:16,407 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:17,135 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:21,408 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:22,766 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:22,767 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:22,767 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:22,768 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:23,138 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:26,769 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:27,224 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:06:27,355 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:27,356 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:27,356 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:27,357 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:28,140 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:28,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:06:28,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:06:32,592 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:34,071 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:34,073 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:34,073 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:34,074 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:34,075 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:34,075 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 1 < 86. Dropping entry: {'loss/epoch': 0.1316830727899837, '_timestamp': 1700211994.0749989}.
+2023-11-17 09:06:34,142 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:36,143 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:06:38,143 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:06:38,416 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:41,144 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:06:43,145 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:06:43,442 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:43,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:06:43,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:06:43,910 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:43,911 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:43,912 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:43,912 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:44,146 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:45,146 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:06:47,147 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:06:48,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:48,384 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:48,384 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:48,385 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:49,148 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:49,385 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:54,386 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:06:54,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:06:54,750 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:06:54,750 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:06:54,750 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:06:55,150 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:06:57,225 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:06:58,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:06:58,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:06:59,613 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:01,846 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:01,848 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:01,848 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:01,849 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:02,153 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:04,850 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:08,224 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:08,225 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:08,226 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:08,226 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:09,156 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:10,227 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:13,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:07:13,466 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:07:14,938 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:14,939 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:14,939 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:14,940 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:15,158 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:15,940 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:20,941 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:21,314 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:21,315 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:21,315 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:21,316 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:22,160 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:26,317 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:27,225 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:07:28,296 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:28,298 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:28,298 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:28,299 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:28,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:07:28,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:07:29,163 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:31,625 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:34,937 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:34,938 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:34,938 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:34,939 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:35,165 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:36,940 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:41,941 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:42,471 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:42,472 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:42,473 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:42,473 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:43,168 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:43,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:07:43,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:07:47,612 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:49,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:49,486 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:49,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:49,487 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:50,174 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:53,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:07:56,747 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:07:56,748 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:07:56,748 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:07:56,749 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:07:57,177 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:07:57,226 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:07:58,900 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:07:58,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:07:59,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:04,022 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:04,293 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:04,294 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:04,294 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:04,295 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:05,180 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:09,296 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:10,643 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:10,644 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:10,644 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:10,645 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:11,182 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:13,900 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:08:13,900 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:08:15,022 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:17,196 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:17,197 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:17,197 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:17,198 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:18,185 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:20,199 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:24,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:24,637 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:24,637 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:24,638 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:25,187 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:25,639 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:27,227 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:08:28,900 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:08:28,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:08:31,074 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:31,679 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:31,680 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:31,680 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:31,681 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:32,190 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:36,682 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:38,218 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:38,220 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:38,220 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:38,220 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:39,192 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:42,221 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:43,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:08:43,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:08:44,972 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:44,973 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:44,973 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:44,974 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:45,200 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:47,975 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:52,948 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:52,949 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:52,949 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:52,950 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:08:53,203 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:08:53,951 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:57,228 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:08:58,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:08:58,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:08:58,997 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:08:59,493 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:08:59,494 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:08:59,494 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:08:59,494 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:00,206 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:04,495 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:06,233 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:06,234 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:06,235 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:06,235 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:07,208 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:10,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:13,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:09:13,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:09:14,660 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:14,661 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:14,661 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:14,662 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:15,211 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:15,663 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:17,332 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:17,333 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:17,333 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:17,334 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:18,213 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:21,335 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:24,732 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:24,734 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:24,734 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:24,734 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:25,215 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:26,735 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:27,229 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:09:28,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:09:28,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:09:31,718 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:31,719 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:31,719 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:31,720 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:32,218 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:32,720 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:37,721 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:39,217 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:39,219 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:39,219 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:39,219 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:39,221 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:43,220 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:43,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:09:43,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:09:45,779 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:45,780 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:45,780 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:45,781 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:46,224 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:48,782 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:52,506 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:52,507 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:52,507 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:52,508 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:09:53,226 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:09:54,509 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:09:57,230 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:09:58,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:09:58,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:09:59,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:09:59,242 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:09:59,242 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:09:59,242 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:00,229 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:00,243 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:05,244 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:05,963 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:05,965 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:05,965 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:05,965 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:06,232 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:08,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:08,760 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:08,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:08,761 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:09,233 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:10,764 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:13,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:10:13,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:10:15,978 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:16,210 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:16,212 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:16,212 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:16,212 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:16,236 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:21,214 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:22,618 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:22,619 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:22,620 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:22,620 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:23,238 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:26,621 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:27,230 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:10:28,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:10:28,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:10:29,857 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:29,858 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:29,858 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:29,859 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:30,249 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:31,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:36,664 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:36,665 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:36,665 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:36,666 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:37,251 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:37,666 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:42,667 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:43,698 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:43,699 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:43,700 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:43,700 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:43,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:10:43,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:10:44,254 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:48,022 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:50,092 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:50,093 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:50,094 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:50,094 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:50,256 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:53,095 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:55,101 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:10:55,104 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:10:55,104 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:10:55,105 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:10:55,258 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:10:57,231 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:10:58,232 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:10:58,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:10:58,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:11:02,602 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:02,603 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:02,603 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:02,604 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:03,262 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:03,605 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:08,606 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:09,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:09,020 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:09,021 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:09,021 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:09,264 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:13,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:11:13,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:11:13,957 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:17,109 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:17,110 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:17,111 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:17,111 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:17,270 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:19,112 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:23,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:23,861 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:23,861 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:23,862 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:24,272 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:24,862 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:27,232 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:11:28,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:11:28,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:11:30,031 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:30,238 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:30,239 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:30,240 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:30,240 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:30,275 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:35,242 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:36,822 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:36,823 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:36,823 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:36,824 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:37,289 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:40,826 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:43,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:11:43,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:11:45,256 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:45,257 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:45,257 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:45,258 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:45,294 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:46,258 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:51,259 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:52,283 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:52,284 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:52,284 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:52,285 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:52,297 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:11:56,286 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:11:57,233 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:11:58,697 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:11:58,698 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:11:58,699 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:11:58,699 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:11:58,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:11:58,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:11:59,300 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:02,034 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:05,417 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:05,418 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:05,419 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:05,419 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:06,303 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:07,420 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:12,421 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:13,418 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:13,419 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:13,419 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:13,420 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:13,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:12:13,902 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:12:14,306 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:17,976 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:19,799 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:19,800 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:19,800 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:19,801 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:20,308 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:23,802 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:27,130 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:27,131 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:27,132 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:27,132 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:27,234 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:12:27,311 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:28,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:12:28,903 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:12:29,045 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:30,039 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:30,041 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:30,041 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:30,041 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:30,312 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:35,043 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:38,511 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:38,512 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:38,512 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:38,513 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:39,315 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:40,514 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:43,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:12:43,903 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:12:45,085 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:45,087 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:45,087 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:45,087 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:45,317 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:46,088 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:51,089 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:52,530 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:52,531 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:52,531 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:52,532 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:53,320 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:12:56,533 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:12:57,235 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:12:58,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:12:58,903 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:12:59,004 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:12:59,005 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:12:59,006 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:12:59,006 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:12:59,322 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:02,007 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:07,008 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:07,190 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:13:07,191 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:13:07,191 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:13:07,191 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:13:07,325 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:12,193 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:13,564 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:13:13,565 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:13:13,566 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:13:13,566 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:13:13,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:13:13,903 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:13:14,328 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:17,975 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:20,615 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:13:20,616 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:13:20,616 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:13:20,617 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:13:21,330 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:23,618 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:27,235 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:13:27,649 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:13:27,651 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:13:27,651 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:13:27,651 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:13:28,333 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:28,652 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:28,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:13:28,903 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:13:34,027 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:34,402 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:13:34,403 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:13:34,404 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:13:34,404 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:13:35,336 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:39,405 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:42,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:13:42,384 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:13:42,384 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:13:42,384 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:13:43,339 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:13:44,152 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:13:44,152 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:13:45,219 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:51,015 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:51,342 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:13:53,343 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:13:55,343 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:13:56,652 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:13:57,236 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:13:57,344 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:13:59,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:13:59,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:13:59,345 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:01,346 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:02,295 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:03,346 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:05,347 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:07,348 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:07,962 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:09,349 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:11,350 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:13,350 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:13,615 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:14,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:14:14,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:14:15,351 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:17,352 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:19,276 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:19,354 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:21,355 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:24,199 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 09:14:24,199 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_0_149_9acbaa99fb15c017cbb0.png with policy now
+2023-11-17 09:14:24,200 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:14:24,201 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:14:24,202 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:14:24,202 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:14:24,361 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:14:24,361 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_149_9acbaa99fb15c017cbb0.png
+2023-11-17 09:14:24,361 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 09:14:24,362 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media
+2023-11-17 09:14:24,416 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:24,815 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/m2mxk0xc-media/images/sample_0_149_9acbaa99fb15c017cbb0.png
+2023-11-17 09:14:25,153 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/c0m1o72h-media/images/sample_0_149_9acbaa99fb15c017cbb0.png
+2023-11-17 09:14:25,361 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:27,237 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:14:27,362 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:29,152 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:14:29,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:14:29,363 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:30,028 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:31,364 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:33,369 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:35,370 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:35,529 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:37,371 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:39,372 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:41,177 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:41,373 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:43,374 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:44,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:14:44,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:14:45,375 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:46,838 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:47,376 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:49,377 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:51,378 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:52,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:53,379 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:55,380 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:57,238 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:14:57,381 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:14:58,238 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:14:58,747 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 09:14:58,747 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_1_150_b7c26fa9bbca9c3cddb4.png with policy now
+2023-11-17 09:14:58,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:14:58,760 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:14:58,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:14:58,761 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:14:58,770 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:14:58,770 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 150 < 151. Dropping entry: {'loss/current': 0.12452153861522675, 'loss/average': 0.11976450161878453, 'lr/unet': 4e-05, '_timestamp': 1700212498.7697875}.
+2023-11-17 09:14:59,005 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/8hkger9q-media/images/sample_1_150_b7c26fa9bbca9c3cddb4.png
+2023-11-17 09:14:59,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:14:59,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:14:59,382 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:14:59,383 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_150_b7c26fa9bbca9c3cddb4.png
+2023-11-17 09:14:59,383 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 09:14:59,660 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/vek0rbya-media/images/sample_1_150_b7c26fa9bbca9c3cddb4.png
+2023-11-17 09:15:03,282 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:05,579 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:08,283 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:11,928 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:11,929 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:11,929 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:11,930 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:12,392 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:13,931 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:14,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:15:14,154 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:15:18,936 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:18,937 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:18,937 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:18,938 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:18,938 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:19,396 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:23,527 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:23,528 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:23,529 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:23,529 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:24,398 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:24,530 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:27,239 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:15:29,043 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:29,045 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:29,045 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:29,046 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:29,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:15:29,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:15:29,401 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:30,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:35,242 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:35,443 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:35,444 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:35,445 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:35,445 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:36,404 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:40,446 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:41,915 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:41,916 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:41,916 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:41,917 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:42,407 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:44,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:15:44,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:15:46,279 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:49,238 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:49,239 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:49,240 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:49,240 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:49,411 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:52,241 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:56,079 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:15:56,080 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:15:56,081 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:15:56,081 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:15:56,414 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:15:57,240 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:15:58,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:15:59,155 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:15:59,155 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:16:02,717 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:02,718 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:02,718 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:02,719 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:03,417 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:03,720 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:08,720 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:09,046 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:09,047 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:09,047 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:09,048 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:09,420 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:14,049 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:14,154 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:16:14,154 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:16:15,706 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:15,707 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:15,708 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:15,708 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:16,424 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:19,709 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:23,174 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:23,176 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:23,176 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:23,177 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:23,427 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:25,177 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:27,240 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:16:29,154 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:16:29,154 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:16:29,613 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:29,614 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:29,614 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:29,615 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:30,430 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:30,616 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:35,617 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:36,837 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:36,838 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:36,838 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:36,839 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:37,434 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:40,840 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:44,153 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:16:44,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:16:44,261 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:44,262 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:44,262 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:44,263 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:44,437 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:46,264 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:50,721 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:50,722 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:50,723 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:50,723 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:51,440 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:51,724 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:56,725 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:16:57,102 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:16:57,103 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:16:57,103 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:16:57,104 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:16:57,241 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:16:57,443 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:16:59,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:16:59,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:17:02,598 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:05,894 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:05,895 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:05,895 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:05,896 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:06,447 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:07,896 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:10,470 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:10,471 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:10,472 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:10,472 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:11,450 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:12,317 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:12,318 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:12,319 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:12,319 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:12,450 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:13,320 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:14,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:17:14,550 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:17:18,691 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:20,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:20,488 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:20,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:20,489 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:20,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:20,490 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 2 < 172. Dropping entry: {'loss/epoch': 0.11759076325959245, '_timestamp': 1700212640.4899392}.
+2023-11-17 09:17:21,455 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:23,456 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:17:23,893 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:25,457 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:17:27,244 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:17:27,458 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:17:29,332 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:29,459 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:17:29,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:17:29,549 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:17:29,969 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:29,971 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:29,971 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:29,972 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:30,460 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:31,460 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:17:33,461 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:17:34,767 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:36,395 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:36,397 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:36,397 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:36,398 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:36,463 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:40,399 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:43,166 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:43,167 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:43,168 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:43,168 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:43,466 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:44,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:17:44,550 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:17:45,638 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:50,639 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:51,590 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:51,591 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:51,592 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:51,593 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:52,471 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:56,596 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:17:57,247 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:17:59,472 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:17:59,473 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:17:59,474 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:17:59,475 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:17:59,477 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:17:59,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:17:59,550 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:18:01,598 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:06,509 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:06,510 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:06,511 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:06,511 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:07,482 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:07,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:12,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:13,129 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:13,130 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:13,130 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:13,131 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:13,485 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:14,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:18:14,550 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:18:17,608 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:19,890 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:19,892 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:19,892 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:19,892 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:20,488 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:22,893 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:24,502 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:24,503 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:24,504 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:24,504 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:25,491 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:27,247 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:18:28,248 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:29,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:18:29,550 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:18:30,867 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:30,871 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:30,872 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:30,872 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:31,494 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:33,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:37,607 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:37,609 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:37,609 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:37,610 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:38,497 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:39,611 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:44,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:18:44,550 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:18:44,628 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:44,725 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:44,726 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:44,726 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:44,727 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:45,501 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:46,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:46,466 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:46,467 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:46,467 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:46,501 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:50,468 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:50,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:50,760 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:50,761 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:50,761 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:51,504 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:55,762 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:18:57,248 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:18:57,914 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:18:57,915 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:18:57,915 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:18:57,916 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:18:58,507 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:18:59,711 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:18:59,711 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:19:01,740 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:04,704 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:04,706 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:04,706 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:04,706 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:05,510 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:07,707 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:12,693 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:12,695 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:12,695 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:12,696 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:13,514 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:13,696 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:14,711 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:19:14,712 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:19:18,839 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:19,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:19,183 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:19,183 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:19,184 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:19,517 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:24,185 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:25,912 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:25,913 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:25,913 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:25,914 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:26,521 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:27,249 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:19:29,250 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:29,712 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:19:29,712 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:19:32,300 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:32,301 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:32,302 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:32,303 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:32,524 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:34,304 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:38,805 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:38,806 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:38,806 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:38,807 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:39,528 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:39,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:44,712 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:19:44,712 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:19:45,740 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:46,197 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:46,198 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:46,198 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:46,199 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:46,531 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:51,200 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:52,545 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:52,546 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:52,547 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:52,547 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:53,535 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:56,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:19:57,258 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:19:58,909 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:19:58,910 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:19:58,910 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:19:58,911 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:19:59,538 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:19:59,712 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:19:59,712 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:20:01,847 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:05,198 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:05,199 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:05,199 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:05,200 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:05,541 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:07,200 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:11,625 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:11,627 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:11,627 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:11,627 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:12,544 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:12,628 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:14,711 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:20:14,712 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:20:17,824 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:18,779 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:18,780 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:18,780 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:18,781 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:19,548 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:23,782 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:27,255 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:20:27,639 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:27,641 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:27,641 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:27,644 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:28,552 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:29,644 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:29,777 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:20:29,778 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:20:34,871 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:35,159 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:35,160 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:35,160 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:35,161 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:35,555 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:40,164 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:42,172 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:42,174 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:42,174 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:42,175 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:42,559 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:44,777 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:20:44,778 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:20:45,858 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:48,776 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:48,778 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:48,779 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:48,779 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:49,562 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:51,666 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:51,667 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:51,668 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:51,668 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:51,669 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:52,564 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:56,670 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:20:57,255 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:20:58,056 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:20:58,058 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:20:58,058 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:20:58,061 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:20:58,567 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:20:59,814 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:20:59,815 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:21:01,932 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:04,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:04,384 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:04,384 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:04,385 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:04,570 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:07,386 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:11,804 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:11,805 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:11,805 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:11,806 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:12,573 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:12,807 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:14,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:21:14,813 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:21:17,916 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:18,063 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:18,064 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:18,064 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:18,065 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:18,576 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:23,066 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:25,088 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:25,089 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:25,090 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:25,090 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:25,579 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:27,256 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:21:28,257 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:29,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:21:29,813 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:21:30,159 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:30,161 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:30,162 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:30,164 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:30,582 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:34,165 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:36,999 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:37,000 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:37,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:37,001 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:37,587 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:40,002 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:43,810 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:43,811 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:43,812 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:43,812 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:44,590 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:44,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:21:44,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:21:45,909 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:50,212 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:50,214 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:50,214 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:50,215 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:50,593 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:51,215 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:56,216 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:21:56,615 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:21:56,616 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:21:56,616 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:21:56,617 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:21:57,257 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:21:57,596 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:21:59,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:21:59,813 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:22:01,897 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:03,317 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:03,318 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:03,318 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:03,321 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:03,599 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:07,322 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:11,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:11,822 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:11,823 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:11,823 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:12,603 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:12,824 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:14,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:22:14,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:22:17,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:20,006 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:20,008 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:20,008 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:20,008 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:20,607 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:23,009 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:27,258 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:22:27,463 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:27,464 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:27,465 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:27,465 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:27,610 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:28,466 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:29,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:22:29,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:22:33,844 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:33,879 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:33,880 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:33,880 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:33,883 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:34,613 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:38,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:41,091 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:41,093 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:41,093 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:41,093 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:41,617 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:44,098 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:44,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:22:44,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:22:48,986 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:48,988 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:48,988 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:48,990 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:49,620 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:49,990 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:54,129 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:22:54,131 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:22:54,131 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:22:54,132 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:22:54,623 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:22:55,132 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:22:57,259 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:22:59,814 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:22:59,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:23:00,473 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:00,475 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:00,475 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:00,476 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:00,476 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:00,626 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:05,477 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:07,846 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:07,847 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:07,847 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:07,850 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:08,630 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:10,851 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:14,814 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:23:14,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:23:14,914 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:14,917 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:14,917 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:14,918 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:15,633 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:15,919 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:20,920 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:21,259 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:21,261 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:21,261 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:21,263 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:21,636 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:26,264 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:27,260 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:23:28,538 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:28,540 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:28,540 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:28,541 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:28,639 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:29,814 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:23:29,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:23:31,948 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:35,258 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:35,259 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:35,259 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:35,260 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:35,642 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:37,261 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:42,262 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:43,231 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:43,232 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:43,232 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:43,235 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:43,646 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:44,998 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:23:44,999 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:23:48,086 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:49,757 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:49,759 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:49,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:49,759 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:50,650 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:53,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:56,569 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:23:56,570 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:23:56,570 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:23:56,571 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:23:56,652 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:23:57,260 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:23:59,261 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:23:59,998 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:23:59,999 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:24:02,945 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:02,946 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:02,947 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:02,947 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:03,656 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:04,948 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:09,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:09,488 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:09,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:09,489 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:09,659 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:10,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:14,999 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:24:15,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:24:16,141 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:16,896 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:16,897 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:16,898 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:16,903 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:17,663 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:21,904 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:24,001 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:24,002 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:24,003 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:24,003 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:24,682 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:27,004 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:27,262 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:24:30,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:24:30,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:24:30,365 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:30,366 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:30,367 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:30,367 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:30,700 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:32,368 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:37,128 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:37,129 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:37,129 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:37,130 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:37,703 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:38,131 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:43,131 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:44,201 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:44,202 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:44,203 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:44,204 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:44,706 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:45,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:24:45,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:24:49,128 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:52,690 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:24:52,691 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:24:52,692 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:24:52,692 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:24:52,716 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:24:54,693 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:24:57,263 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:24:59,999 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:00,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:25:00,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:25:00,117 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:00,118 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:00,119 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:00,121 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:00,720 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:05,122 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:06,472 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:06,473 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:06,473 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:06,474 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:06,723 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:10,475 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:12,832 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:12,833 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:12,833 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:12,834 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:13,726 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:15,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:25:15,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:25:16,101 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:19,504 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:19,506 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:19,506 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:19,506 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:19,729 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:21,507 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:26,249 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:26,250 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:26,250 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:26,253 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:26,732 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:27,253 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:27,263 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:25:30,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:25:30,001 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:25:33,101 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:33,267 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:33,268 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:33,268 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:33,269 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:33,736 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:38,270 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:39,833 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:39,834 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:39,834 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:39,835 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:40,739 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:43,836 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:45,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:25:45,001 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:25:46,593 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:46,594 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:46,595 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:46,595 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:46,742 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:49,596 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:52,982 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:52,983 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:52,984 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:52,984 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:53,746 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:54,985 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:25:57,268 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:25:59,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:25:59,550 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:25:59,551 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:25:59,551 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:25:59,749 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:25:59,999 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:00,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:26:00,000 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:26:05,078 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:06,993 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:06,995 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:06,995 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:06,996 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:07,753 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:10,997 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:12,983 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:12,985 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:12,985 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:12,986 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:13,756 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:15,235 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:26:15,236 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:26:16,365 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:21,165 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:21,166 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:21,166 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:21,167 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:21,761 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:22,167 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:27,168 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:27,268 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:26:29,509 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:29,511 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:29,511 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:29,514 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:29,764 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:30,235 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:26:30,236 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:26:32,278 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:36,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:36,551 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:36,551 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:36,552 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:36,768 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:37,552 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:40,954 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:40,955 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:40,956 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:40,956 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:41,770 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:42,957 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:43,699 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:43,700 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:43,700 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:43,701 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:43,771 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:45,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:26:45,236 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:26:46,593 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:46,595 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:46,595 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:46,595 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:46,773 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:48,596 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:53,597 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:26:53,942 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:53,943 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:26:53,943 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:26:53,944 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:26:53,944 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:26:53,944 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 3 < 258. Dropping entry: {'loss/epoch': 0.11971940539863914, '_timestamp': 1700213213.942237}.
+2023-11-17 09:26:54,777 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:26:55,777 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:26:57,268 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:26:57,778 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:26:58,779 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:26:58,906 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:00,235 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:27:00,236 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:27:00,780 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:27:02,177 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:02,178 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:02,178 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:02,179 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:02,781 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:27:02,781 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:04,378 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:04,782 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:27:09,379 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:10,086 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:10,087 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:10,088 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:10,088 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:10,785 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:15,089 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:15,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:27:15,237 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:27:16,785 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:16,786 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:16,786 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:16,787 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:16,788 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:20,789 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:23,541 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:23,542 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:23,542 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:23,543 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:23,791 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:26,544 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:27,269 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:27:30,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:27:30,236 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:27:30,796 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:30,797 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:30,798 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:30,800 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:31,794 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:31,801 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:36,802 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:37,501 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:37,502 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:37,503 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:37,503 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:37,797 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:42,504 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:43,909 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:43,910 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:43,911 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:43,911 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:44,801 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:45,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:27:45,237 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:27:48,324 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:51,129 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:51,130 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:51,130 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:51,131 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:51,805 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:53,367 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:53,368 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:53,369 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:53,369 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:53,370 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:53,807 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:27:57,270 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:27:59,271 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:27:59,681 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:27:59,682 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:27:59,683 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:27:59,683 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:27:59,812 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:00,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:28:00,237 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:28:04,331 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:06,051 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:06,053 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:06,053 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:06,055 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:06,816 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:10,057 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:12,589 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:12,590 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:12,590 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:12,591 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:12,819 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:15,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:28:15,237 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:28:15,301 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:20,302 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:20,928 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:20,929 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:20,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:20,930 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:21,823 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:25,931 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:27,271 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:28:27,331 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:27,332 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:27,332 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:27,333 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:27,826 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:30,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:28:30,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:28:31,365 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:35,073 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:35,074 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:35,075 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:35,075 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:35,830 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:37,076 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:42,077 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:42,941 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:42,942 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:42,943 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:42,945 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:43,834 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:45,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:28:45,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:28:45,828 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:45,829 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:45,829 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:45,830 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:45,835 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:47,831 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:52,214 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:52,215 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:52,216 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:52,216 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:52,839 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:53,217 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:28:57,164 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:28:57,169 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:28:57,170 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:28:57,170 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:28:57,272 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:28:57,841 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:28:58,272 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:00,037 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:00,038 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:00,038 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:00,039 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:00,245 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:29:00,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:29:00,843 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:03,350 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:06,778 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:06,779 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:06,780 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:06,780 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:06,847 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:08,781 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:11,086 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:11,087 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:11,088 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:11,088 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:11,850 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:14,093 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:15,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:29:15,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:29:17,414 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:17,415 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:17,415 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:17,418 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:17,853 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:19,419 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:24,420 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:24,858 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:24,859 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:24,859 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:24,860 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:25,856 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:27,272 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:29:30,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:29:30,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:29:30,352 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:32,824 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:32,826 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:32,826 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:32,826 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:32,859 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:35,339 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:35,343 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:35,343 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:35,344 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:35,861 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:36,347 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:41,352 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:41,877 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:41,878 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:41,879 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:41,879 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:42,865 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:45,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:29:45,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:29:47,329 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:50,149 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:50,150 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:50,150 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:50,153 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:50,868 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:53,157 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:29:57,187 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:29:57,189 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:29:57,189 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:29:57,189 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:29:57,273 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:29:57,872 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:29:58,274 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:00,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:30:00,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:30:03,338 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:04,377 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:04,378 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:04,378 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:04,379 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:04,880 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:08,380 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:11,071 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:11,072 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:11,073 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:11,073 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:11,883 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:14,074 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:15,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:30:15,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:30:17,410 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:17,411 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:17,411 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:17,412 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:17,886 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:19,413 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:23,886 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:23,887 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:23,887 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:23,892 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:23,892 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:24,893 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:27,274 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:30:30,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:30:30,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:30:30,305 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:31,257 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:31,258 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:31,259 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:31,259 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:31,896 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:36,261 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:37,620 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:37,621 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:37,621 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:37,622 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:37,899 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:41,623 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:43,577 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:43,578 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:43,578 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:43,579 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:43,902 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:45,563 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:30:45,563 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:30:46,690 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:51,001 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:51,002 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:51,003 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:51,003 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:51,906 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:30:52,004 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:57,005 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:30:57,275 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:30:58,972 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:30:58,973 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:30:58,974 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:30:58,974 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:30:59,910 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:31:00,747 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:31:00,747 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:31:02,848 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:05,346 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:31:05,347 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:31:05,347 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:31:05,348 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:31:05,912 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:31:08,349 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:11,993 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:31:11,994 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:31:11,994 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:31:11,995 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:31:12,915 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:31:13,995 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:15,747 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:31:15,748 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:31:18,684 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:31:18,685 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:31:18,686 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:31:18,686 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:31:18,918 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:31:19,687 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:24,688 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:25,923 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:26,923 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:27,275 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:31:28,924 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:29,724 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:30,747 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:31:30,748 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:31:30,925 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:32,926 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:34,927 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:35,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:36,928 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:38,929 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:40,910 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:40,930 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:42,931 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:44,932 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:45,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:31:45,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:31:46,701 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:46,933 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:48,934 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:50,935 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:52,231 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:52,936 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:54,937 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:56,938 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:31:57,276 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:31:57,279 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:31:58,939 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:00,311 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 09:32:00,312 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_0_299_21ebb3119669773f9840.png with policy now
+2023-11-17 09:32:00,318 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:00,325 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:32:00,326 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:32:00,328 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:32:00,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:32:00,750 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:32:00,906 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/zngp38h4-media/images/sample_0_299_21ebb3119669773f9840.png
+2023-11-17 09:32:00,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:32:00,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_299_21ebb3119669773f9840.png
+2023-11-17 09:32:00,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 09:32:01,229 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/rdrbn4wu-media/images/sample_0_299_21ebb3119669773f9840.png
+2023-11-17 09:32:01,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:02,673 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:02,941 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:04,942 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:06,943 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:08,241 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:08,944 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:10,945 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:12,946 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:13,870 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:14,947 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:15,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:32:15,748 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:32:16,948 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:18,949 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:19,502 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:20,950 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:22,951 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:24,952 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:25,150 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:26,953 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:27,277 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:32:28,954 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:30,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:30,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:32:30,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:32:30,955 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:32,956 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:34,831 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 09:32:34,832 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_1_300_9c81ac6c681cfe260b5e.png with policy now
+2023-11-17 09:32:34,832 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:34,837 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:32:34,838 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:32:34,838 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:32:34,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:34,860 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 300 < 301. Dropping entry: {'loss/current': 0.2046741545200348, 'loss/average': 0.11499301919193808, 'lr/unet': 4e-05, '_timestamp': 1700213554.858412}.
+2023-11-17 09:32:34,959 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:32:34,959 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:32:34,960 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_300_9c81ac6c681cfe260b5e.png
+2023-11-17 09:32:34,961 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 09:32:35,090 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/bcewh3qy-media/images/sample_1_300_9c81ac6c681cfe260b5e.png
+2023-11-17 09:32:35,428 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/3kl36jsw-media/images/sample_1_300_9c81ac6c681cfe260b5e.png
+2023-11-17 09:32:35,839 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:40,839 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:41,654 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:45,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:32:45,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:32:45,874 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:47,997 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:47,998 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:32:47,999 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:32:47,999 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:32:48,970 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:32:51,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:53,082 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:53,083 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:32:53,084 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:32:53,084 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:32:53,972 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:32:56,085 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:32:57,278 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:32:59,592 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:32:59,593 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:32:59,593 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:32:59,594 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:32:59,975 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:00,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:33:00,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:33:01,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:06,219 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:06,220 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:06,220 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:06,221 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:06,979 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:07,222 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:12,222 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:14,556 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:14,557 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:14,558 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:14,558 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:14,982 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:15,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:33:15,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:33:17,849 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:20,514 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:20,515 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:20,515 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:20,516 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:20,985 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:23,530 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:27,278 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:33:27,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:27,886 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:27,886 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:27,886 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:27,989 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:28,887 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:30,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:33:30,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:33:34,249 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:34,250 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:34,251 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:34,251 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:34,251 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:34,992 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:39,252 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:42,865 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:42,866 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:42,867 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:42,867 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:42,996 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:44,868 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:45,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:33:45,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:33:50,205 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:50,206 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:50,207 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:50,207 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:50,208 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:51,000 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:33:55,209 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:33:57,279 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:33:58,216 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:33:58,218 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:33:58,218 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:33:58,219 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:33:59,004 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:00,219 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:00,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:34:00,750 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:34:05,815 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:06,858 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:06,863 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:06,863 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:06,863 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:07,008 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:10,864 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:15,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:15,663 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:15,664 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:15,664 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:15,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:34:15,749 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:34:15,892 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:16,012 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:20,893 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:22,039 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:22,040 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:22,040 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:22,041 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:23,015 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:26,042 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:26,774 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:26,775 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:26,776 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:26,776 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:27,017 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:27,280 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:34:30,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:34:30,751 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:34:31,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:35,122 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:35,123 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:35,124 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:35,124 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:36,022 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:37,125 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:41,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:41,513 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:41,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:41,514 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:42,025 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:42,515 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:45,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:34:45,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:34:47,886 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:48,072 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:48,073 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:48,073 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:48,073 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:49,028 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:53,074 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:55,154 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:55,155 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:55,156 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:55,156 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:34:56,047 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:34:57,280 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:34:58,281 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:34:59,820 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:34:59,821 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:34:59,821 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:34:59,822 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:00,049 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:00,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:35:00,751 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:35:03,801 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:06,525 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:06,527 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:06,527 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:06,528 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:07,053 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:09,529 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:13,720 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:13,721 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:13,722 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:13,722 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:14,057 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:14,723 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:15,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:35:15,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:35:19,797 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:20,432 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:20,433 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:20,434 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:20,434 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:21,060 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:25,435 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:27,281 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:35:27,471 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:27,472 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:27,473 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:27,473 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:28,064 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:30,474 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:30,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:35:30,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:35:34,023 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:34,024 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:34,025 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:34,025 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:34,067 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:36,026 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:40,737 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:40,738 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:40,738 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:40,739 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:41,070 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:41,739 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:45,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:35:45,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:35:46,802 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:47,731 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:47,733 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:47,733 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:47,734 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:48,074 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:52,735 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:35:54,914 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:35:54,915 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:35:54,916 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:35:54,916 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:35:55,077 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:35:57,282 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:35:58,283 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:00,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:36:00,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:36:02,311 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:02,312 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:02,312 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:02,313 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:03,081 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:03,313 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:08,314 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:09,335 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:09,337 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:09,337 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:09,337 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:10,085 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:13,339 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:15,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:36:15,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:36:16,518 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:16,519 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:16,519 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:16,520 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:17,088 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:18,526 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:23,241 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:23,242 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:23,242 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:23,243 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:24,092 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:24,243 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:27,283 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:36:29,284 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:29,605 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:29,606 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:29,607 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:29,607 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:30,095 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:30,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:36:30,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:36:34,782 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:36,801 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:36,803 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:36,803 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:36,803 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:37,098 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:39,804 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:43,564 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:43,565 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:43,565 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:43,566 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:44,102 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:45,566 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:45,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:36:45,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:36:49,911 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:49,912 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:49,912 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:49,913 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:50,105 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:50,913 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:55,914 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:36:56,972 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:36:56,973 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:36:56,974 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:36:56,974 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:36:57,108 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:36:57,284 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:37:00,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:37:00,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:37:01,872 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:03,706 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:03,708 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:03,708 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:03,709 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:04,112 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:07,710 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:10,717 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:10,718 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:10,718 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:10,719 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:11,115 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:12,720 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:15,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:37:15,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:37:17,702 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:17,703 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:17,703 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:17,704 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:18,123 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:18,704 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:23,705 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:24,327 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:24,328 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:24,328 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:24,329 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:25,127 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:27,284 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:37:29,285 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:30,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:37:30,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:37:31,428 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:31,429 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:31,429 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:31,430 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:32,130 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:34,431 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:37,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:37,755 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:37,756 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:37,756 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:37,757 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:37,757 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 4 < 344. Dropping entry: {'loss/epoch': 0.11610582549947986, '_timestamp': 1700213857.7547774}.
+2023-11-17 09:37:38,133 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:40,134 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:37:40,392 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:43,138 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:37:45,139 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:37:45,462 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:45,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:37:45,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:37:46,218 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:46,220 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:46,220 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:46,221 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:47,140 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:37:47,140 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:49,141 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:37:51,318 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:54,136 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:37:54,137 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:37:54,138 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:37:54,139 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:37:54,143 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:37:57,140 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:37:57,285 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:38:00,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:38:00,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:38:02,099 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:02,100 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:02,101 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:02,101 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:02,147 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:03,102 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:08,105 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:09,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:09,486 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:09,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:09,487 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:10,152 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:13,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:15,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:38:15,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:38:15,870 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:15,871 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:15,871 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:15,872 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:16,155 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:18,872 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:22,236 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:22,237 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:22,237 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:22,238 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:23,158 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:24,238 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:27,286 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:38:29,287 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:29,574 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:29,575 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:29,576 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:29,576 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:30,161 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:30,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:38:30,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:38:34,849 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:36,304 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:36,305 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:36,305 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:36,306 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:37,165 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:40,307 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:44,265 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:44,266 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:44,266 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:44,267 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:45,169 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:45,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:45,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:38:45,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:38:50,804 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:51,537 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:51,538 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:51,538 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:51,539 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:52,172 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:38:56,540 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:38:57,287 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:38:58,180 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:38:58,182 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:38:58,182 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:38:58,186 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:38:59,175 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:00,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:39:00,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:39:01,852 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:05,596 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:05,597 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:05,597 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:05,598 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:06,179 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:07,599 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:12,600 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:14,219 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:14,220 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:14,220 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:14,221 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:15,183 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:16,018 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:39:16,018 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:39:18,103 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:21,001 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:21,003 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:21,003 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:21,004 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:21,186 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:24,005 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:27,288 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:39:27,397 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:27,398 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:27,398 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:27,399 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:28,190 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:29,128 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:29,129 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:29,130 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:29,130 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:29,130 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:29,190 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:31,018 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:39:31,018 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:39:35,125 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:35,706 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:35,707 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:35,707 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:35,708 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:36,194 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:40,709 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:42,432 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:42,433 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:42,434 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:42,434 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:43,197 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:46,018 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:46,020 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:39:46,020 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:39:49,458 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:49,463 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:49,464 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:49,464 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:50,200 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:51,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:55,976 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:39:55,977 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:39:55,977 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:39:55,977 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:39:56,203 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:39:56,978 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:39:57,288 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:40:01,018 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:40:01,019 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:40:02,105 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:02,286 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:02,288 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:02,288 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:02,288 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:03,207 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:07,290 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:09,310 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:09,311 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:09,311 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:09,312 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:10,211 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:12,312 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:16,018 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:40:16,019 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:40:17,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:17,001 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:17,001 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:17,002 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:17,214 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:18,003 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:23,003 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:23,302 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:23,304 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:23,304 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:23,304 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:24,218 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:26,401 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:26,402 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:26,402 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:26,403 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:27,219 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:27,289 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:40:28,290 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:31,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:40:31,019 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:40:32,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:32,753 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:32,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:32,754 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:33,222 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:33,755 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:38,756 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:41,373 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:41,374 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:41,374 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:41,375 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:42,227 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:44,376 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:46,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:40:46,019 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:40:48,155 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:48,156 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:48,157 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:48,157 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:48,232 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:50,158 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:54,524 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:54,525 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:54,526 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:54,526 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:55,236 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:40:55,527 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:40:57,290 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:40:58,817 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:40:58,818 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:40:58,819 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:40:58,819 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:40:59,238 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:00,820 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:01,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:41:01,019 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:41:05,537 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:05,539 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:05,539 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:05,539 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:06,241 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:06,540 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:11,541 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:11,964 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:11,965 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:11,965 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:11,966 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:12,244 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:16,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:41:16,020 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:41:17,077 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:18,341 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:18,342 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:18,343 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:18,343 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:19,248 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:22,344 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:24,479 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:24,480 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:24,481 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:24,481 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:25,251 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:27,291 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:41:28,291 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:31,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:41:31,020 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:41:31,154 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:31,155 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:31,156 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:31,156 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:31,254 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:34,157 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:37,552 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:37,553 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:37,553 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:37,554 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:38,258 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:39,555 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:44,556 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:45,179 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:45,180 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:45,180 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:45,181 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:45,264 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:46,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:41:46,020 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:41:50,065 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:52,620 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:52,621 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:52,621 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:52,622 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:53,278 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:55,623 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:41:57,205 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:41:57,206 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:41:57,207 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:41:57,207 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:41:57,280 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:41:57,291 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:42:01,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:01,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:42:01,020 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:42:05,249 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:05,250 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:05,250 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:05,251 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:05,289 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:06,252 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:11,252 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:13,646 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:13,647 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:13,648 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:13,648 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:14,293 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:16,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:42:16,021 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:42:17,071 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:20,015 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:20,016 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:20,017 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:20,017 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:20,296 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:23,018 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:27,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:27,022 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:27,022 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:27,023 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:27,292 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:42:27,300 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:28,293 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:31,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:42:31,022 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:42:33,781 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:33,782 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:33,782 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:33,783 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:33,783 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:34,303 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:38,784 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:40,853 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:40,854 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:40,854 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:40,855 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:41,307 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:43,856 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:46,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:42:46,022 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:42:47,838 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:47,839 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:47,840 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:47,840 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:48,310 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:49,841 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:50,579 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:50,580 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:50,580 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:50,580 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:42:51,312 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:42:55,582 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:42:57,297 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:42:59,438 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:42:59,439 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:42:59,439 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:42:59,440 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:00,316 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:01,231 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:43:01,231 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:43:01,296 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:05,807 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:05,809 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:05,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:05,809 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:06,319 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:06,810 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:11,811 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:12,953 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:12,954 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:12,954 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:12,955 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:13,323 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:16,231 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:43:16,232 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:43:17,308 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:19,684 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:19,685 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:19,685 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:19,686 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:20,326 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:22,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:26,059 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:26,060 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:26,060 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:26,061 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:26,329 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:27,294 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:43:28,295 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:31,232 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:43:31,233 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:43:32,398 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:32,399 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:32,400 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:32,402 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:33,333 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:33,403 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:38,404 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:39,799 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:39,800 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:39,801 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:39,801 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:40,336 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:43,803 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:46,232 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:43:46,233 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:43:46,834 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:46,837 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:46,837 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:46,838 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:47,340 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:48,839 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:53,206 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:53,207 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:53,207 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:53,208 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:43:53,343 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:43:54,209 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:57,295 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:43:59,296 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:43:59,866 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:43:59,867 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:43:59,868 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:43:59,868 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:00,346 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:01,232 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:44:01,232 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:44:04,314 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:06,586 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:06,587 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:06,587 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:06,590 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:07,350 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:09,591 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:13,639 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:13,641 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:13,641 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:13,642 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:14,353 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:14,642 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:16,234 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:44:16,234 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:44:20,147 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:20,148 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:20,148 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:20,149 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:20,149 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:20,356 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:25,150 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:26,876 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:26,878 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:26,878 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:26,879 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:27,296 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:44:27,359 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:30,297 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:31,233 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:44:31,234 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:44:33,899 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:33,901 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:33,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:33,901 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:34,363 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:35,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:40,903 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:41,165 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:41,166 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:41,166 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:41,168 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:41,366 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:46,173 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:46,233 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:44:46,234 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:44:48,612 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:48,613 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:48,613 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:48,614 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:49,370 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:51,615 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:54,995 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:44:54,996 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:44:54,996 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:44:54,997 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:44:55,373 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:44:56,998 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:44:57,296 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:45:01,233 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:45:01,234 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:45:01,600 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:01,601 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:01,601 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:01,602 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:02,377 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:02,603 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:07,604 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:08,327 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:08,329 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:08,329 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:08,329 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:08,380 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:13,330 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:15,830 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:15,831 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:15,831 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:15,834 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:16,233 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:45:16,234 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:45:16,384 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:18,367 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:22,311 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:22,312 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:22,313 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:22,313 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:22,387 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:24,314 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:27,297 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:45:29,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:29,688 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:29,688 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:29,688 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:29,689 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:30,427 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:31,234 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:45:31,234 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:45:32,520 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:32,522 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:32,522 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:32,523 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:33,429 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:35,524 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:37,643 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:37,645 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:37,645 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:37,646 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:38,431 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:40,647 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:44,870 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:44,871 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:44,871 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:44,872 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:45,435 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:45,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:46,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:45:46,240 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:45:51,377 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:53,237 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:53,238 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:53,238 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:53,241 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:45:53,439 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:45:57,242 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:45:57,298 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:45:59,614 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:45:59,615 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:45:59,615 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:45:59,616 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:00,442 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:01,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:46:01,240 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:46:02,308 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:07,309 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:07,768 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:07,769 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:07,769 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:07,770 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:08,446 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:12,546 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:12,547 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:12,548 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:12,548 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:12,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:13,449 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:16,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:46:16,240 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:46:18,356 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:19,573 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:19,574 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:19,574 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:19,575 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:20,453 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:23,576 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:25,937 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:25,939 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:25,939 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:25,941 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:26,456 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:27,299 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:46:29,300 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:30,894 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:30,895 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:30,895 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:30,896 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:31,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:46:31,241 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:46:31,458 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:35,287 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:37,609 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:37,610 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:37,611 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:37,611 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:38,462 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:40,612 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:44,026 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:44,028 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:44,028 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:44,028 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:44,465 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:46,029 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:46,240 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:46:46,241 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:46:50,859 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:50,860 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:50,860 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:50,861 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:51,468 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:46:51,862 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:56,862 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:46:57,299 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:46:57,603 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:46:57,605 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:46:57,605 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:46:57,605 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:46:58,472 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:01,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:47:01,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:47:02,328 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:03,981 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:03,982 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:03,982 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:03,983 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:04,475 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:07,984 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:11,032 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:11,033 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:11,034 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:11,034 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:11,035 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:11,035 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 5 < 430. Dropping entry: {'loss/epoch': 0.12599827362181143, '_timestamp': 1700214431.032903}.
+2023-11-17 09:47:11,478 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:12,479 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:47:13,036 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:14,480 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:47:16,245 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:47:16,245 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:47:17,483 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:47:18,177 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:19,484 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:47:21,485 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:47:22,114 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:22,116 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:22,116 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:22,117 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:22,486 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:23,486 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:47:24,117 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:27,300 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:47:28,496 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:28,498 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:28,498 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:28,499 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:29,489 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:29,499 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:31,245 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:47:31,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:47:35,007 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:35,008 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:35,008 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:35,009 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:35,009 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:35,492 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:40,010 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:41,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:41,761 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:41,762 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:41,762 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:42,496 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:45,763 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:46,245 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:47:46,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:47:48,317 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:48,318 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:48,319 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:48,319 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:48,499 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:51,320 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:53,107 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:53,108 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:53,109 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:53,109 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:53,502 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:47:57,110 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:47:57,301 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:47:59,237 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:47:59,239 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:47:59,239 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:47:59,240 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:47:59,505 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:01,245 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:48:01,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:48:02,339 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:05,962 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:05,964 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:05,964 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:05,964 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:06,508 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:07,965 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:12,966 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:13,451 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:13,453 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:13,453 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:13,454 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:13,512 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:16,245 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:48:16,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:48:18,334 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:20,617 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:20,619 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:20,619 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:20,619 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:21,516 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:23,620 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:26,911 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:26,913 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:26,913 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:26,913 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:27,302 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:48:27,519 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:29,302 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:31,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:48:31,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:48:34,366 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:34,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:34,861 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:34,861 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:34,862 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:35,523 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:39,863 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:40,094 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:40,096 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:40,096 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:40,096 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:40,525 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:45,098 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:46,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:48:46,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:48:46,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:46,809 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:46,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:46,810 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:47,532 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:50,811 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:53,174 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:48:53,176 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:48:53,176 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:48:53,176 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:48:53,535 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:48:56,177 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:48:57,302 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:49:00,326 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:49:00,328 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:49:00,328 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:49:00,329 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:49:00,539 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:49:01,234 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:01,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:49:01,246 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:49:06,366 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:07,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:49:07,762 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:49:07,762 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:49:07,762 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:49:08,543 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:49:11,767 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:14,179 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:49:14,180 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:49:14,180 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:49:14,181 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:49:14,546 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:49:16,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:49:16,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:49:17,341 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:20,522 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:49:20,523 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:49:20,524 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:49:20,524 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:49:20,549 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:49:22,525 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:27,303 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:49:28,402 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:29,554 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:31,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:49:31,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:49:31,555 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:33,556 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:34,044 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:35,557 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:37,558 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:39,559 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:39,698 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:41,560 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:43,561 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:45,358 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:45,562 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:46,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:49:46,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:49:47,563 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:49,564 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:51,017 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:51,565 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:53,566 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:55,567 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:56,679 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:49:57,304 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:49:57,568 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:49:59,570 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:01,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:50:01,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:50:02,212 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 09:50:02,213 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_0_449_5d64336bece18f607be4.png with policy now
+2023-11-17 09:50:02,214 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:50:02,217 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:50:02,218 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:50:02,218 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:02,219 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:50:02,571 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:50:02,572 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_449_5d64336bece18f607be4.png
+2023-11-17 09:50:02,572 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 09:50:02,708 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/sqkla7g4-media/images/sample_0_449_5d64336bece18f607be4.png
+2023-11-17 09:50:02,995 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/npza7pfg-media/images/sample_0_449_5d64336bece18f607be4.png
+2023-11-17 09:50:03,572 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:05,573 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:07,574 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:07,839 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:09,575 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:11,576 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:13,475 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:13,577 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:15,578 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:16,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:50:16,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:50:17,579 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:19,130 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:19,580 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:21,581 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:23,582 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:24,767 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:25,584 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:27,304 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:50:27,585 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:29,586 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:30,407 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:31,247 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:50:31,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:50:31,587 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:33,588 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:35,589 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:50:35,918 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:37,277 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 09:50:37,277 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_1_450_96ba8a2d2129a7a35156.png with policy now
+2023-11-17 09:50:37,277 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:50:37,284 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:50:37,284 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:50:37,284 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:50:37,306 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:50:37,306 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 450 < 451. Dropping entry: {'loss/current': 0.08791681379079819, 'loss/average': 0.12586366139395638, 'lr/unet': 4e-05, '_timestamp': 1700214637.3061123}.
+2023-11-17 09:50:37,577 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/bapmpqco-media/images/sample_1_450_96ba8a2d2129a7a35156.png
+2023-11-17 09:50:37,590 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:50:37,590 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_450_96ba8a2d2129a7a35156.png
+2023-11-17 09:50:37,591 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 09:50:37,877 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/rui2138r-media/images/sample_1_450_96ba8a2d2129a7a35156.png
+2023-11-17 09:50:41,285 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:44,718 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:50:46,286 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:46,483 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:50:46,484 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:50:51,121 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:50:51,122 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:50:51,123 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:50:51,123 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:50:51,597 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:50:52,124 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:57,125 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:50:57,305 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:50:57,820 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:50:57,821 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:50:57,822 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:50:57,822 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:50:58,601 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:01,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:51:01,484 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:51:02,534 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:04,155 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:04,156 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:04,156 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:04,157 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:04,604 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:05,889 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:05,891 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:05,891 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:05,892 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:06,605 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:07,892 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:12,893 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:13,027 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:13,029 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:13,029 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:13,030 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:13,609 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:16,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:51:16,484 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:51:18,594 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:19,723 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:19,724 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:19,725 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:19,725 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:20,613 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:23,727 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:26,744 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:26,746 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:26,746 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:26,746 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:27,306 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:51:27,616 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:29,307 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:31,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:51:31,484 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:51:34,278 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:34,279 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:34,279 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:34,280 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:34,620 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:35,281 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:40,282 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:41,667 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:41,668 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:41,669 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:41,669 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:42,624 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:45,670 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:46,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:51:46,484 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:51:48,066 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:48,067 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:48,067 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:48,068 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:48,627 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:51,069 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:54,428 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:51:54,429 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:51:54,429 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:51:54,430 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:51:54,631 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:51:56,430 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:51:57,307 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:52:01,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:52:01,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:52:01,610 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:02,778 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:02,779 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:02,779 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:02,780 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:03,636 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:06,781 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:09,157 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:09,158 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:09,159 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:09,159 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:09,639 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:12,160 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:15,507 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:15,508 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:15,508 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:15,510 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:15,644 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:16,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:52:16,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:52:17,517 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:22,517 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:23,837 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:23,838 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:23,839 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:23,839 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:24,649 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:27,308 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:52:28,308 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:28,545 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:28,546 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:28,546 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:28,547 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:28,651 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:31,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:52:31,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:52:33,617 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:35,544 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:35,545 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:35,546 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:35,546 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:35,655 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:39,547 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:42,784 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:42,785 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:42,785 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:42,786 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:43,659 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:44,787 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:46,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:52:46,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:52:49,517 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:49,518 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:49,519 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:49,519 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:49,662 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:52:50,520 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:55,521 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:52:57,308 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:52:57,491 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:52:57,492 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:52:57,493 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:52:57,493 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:52:57,666 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:00,154 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:00,156 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:00,156 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:00,156 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:00,668 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:01,157 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:01,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:53:01,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:53:06,532 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:09,062 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:09,064 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:09,064 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:09,065 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:09,672 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:12,066 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:15,459 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:15,460 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:15,460 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:15,461 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:15,675 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:16,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:53:16,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:53:17,622 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:22,440 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:22,441 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:22,442 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:22,442 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:22,685 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:23,443 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:27,174 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:27,175 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:27,176 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:27,176 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:27,309 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:53:27,688 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:29,310 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:31,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:53:31,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:53:33,944 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:33,945 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:33,946 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:33,946 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:34,692 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:34,947 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:39,948 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:40,350 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:40,351 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:40,352 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:40,352 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:40,695 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:45,356 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:46,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:53:46,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:53:46,913 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:46,914 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:46,914 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:46,914 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:47,698 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:50,915 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:54,351 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:53:54,353 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:53:54,353 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:53:54,354 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:53:54,702 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:53:56,354 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:53:57,310 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:54:00,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:00,753 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:00,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:00,754 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:01,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:54:01,485 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:54:01,589 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:01,706 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:06,590 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:07,971 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:07,972 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:07,972 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:07,973 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:08,709 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:11,974 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:15,067 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:15,069 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:15,069 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:15,069 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:15,713 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:16,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:54:16,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:54:17,613 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:21,461 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:21,463 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:21,463 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:21,464 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:21,716 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:23,464 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:25,889 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:25,890 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:25,890 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:25,891 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:26,719 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:27,311 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:54:29,311 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:31,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:54:31,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:54:33,084 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:33,086 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:33,086 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:33,087 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:33,722 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:35,088 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:40,088 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:40,453 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:40,455 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:40,455 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:40,461 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:40,726 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:45,463 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:46,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:54:46,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:54:46,835 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:46,837 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:46,837 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:46,838 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:47,730 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:50,839 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:53,386 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:54:53,387 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:54:53,387 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:54:53,388 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:54:53,733 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:54:56,393 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:54:57,312 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:55:00,778 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:00,780 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:00,780 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:00,780 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:01,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:55:01,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:55:01,555 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:01,737 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:06,555 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:08,758 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:08,759 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:08,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:08,760 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:09,741 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:11,761 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:15,301 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:15,302 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:15,303 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:15,303 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:15,744 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:16,485 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:55:16,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:55:17,522 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:22,523 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:24,043 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:24,044 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:24,045 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:24,045 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:24,749 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:27,312 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:55:28,313 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:30,371 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:30,373 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:30,373 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:30,373 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:30,752 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:31,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:55:31,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:55:33,612 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:36,914 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:36,915 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:36,916 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:36,916 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:37,755 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:38,917 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:39,889 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:39,891 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:39,891 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:39,892 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:40,757 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:44,893 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:46,278 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:46,280 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:46,280 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:46,281 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:46,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:55:46,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:55:46,762 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:50,589 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:52,643 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:52,644 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:52,645 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:52,653 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:52,765 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:55:55,653 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:55:57,313 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:55:59,184 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:55:59,185 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:55:59,185 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:55:59,186 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:55:59,769 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:01,187 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:01,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:56:01,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:56:05,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:05,489 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:05,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:05,489 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:05,772 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:06,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:11,491 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:11,827 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:11,829 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:11,829 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:11,829 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:12,776 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:16,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:56:16,486 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:56:16,537 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:19,571 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:19,572 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:19,572 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:19,573 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:19,780 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:21,573 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:22,259 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:22,262 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:22,262 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:22,263 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:22,781 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:27,264 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:27,314 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:56:30,244 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:30,245 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:30,245 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:30,246 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:30,786 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:31,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:56:31,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:56:32,534 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:37,534 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:37,658 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:37,660 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:37,660 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:37,660 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:37,789 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:42,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:46,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:56:46,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:56:46,524 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:46,610 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:46,611 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:46,612 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:46,798 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:48,613 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:53,277 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:56:53,278 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:56:53,278 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:56:53,279 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:56:53,802 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:56:54,279 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:56:57,315 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:56:59,316 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:01,222 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:01,227 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:01,227 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:01,228 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:01,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:57:01,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:57:01,806 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:04,530 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:07,774 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:07,775 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:07,775 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:07,776 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:07,814 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:09,776 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:14,777 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:15,141 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:15,142 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:15,143 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:15,148 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:15,818 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:16,486 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:57:16,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:57:19,801 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:22,157 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:22,158 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:22,159 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:22,159 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:22,822 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:25,160 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:27,315 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:57:28,658 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:28,659 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:28,659 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:28,660 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:28,825 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:30,660 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:31,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:57:31,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:57:36,061 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:36,063 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:36,063 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:36,063 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:36,064 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:36,830 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:41,065 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:43,098 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:43,099 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:43,100 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:43,100 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:43,833 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:46,101 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:46,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:57:46,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:57:49,597 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:49,598 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:49,598 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:49,599 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:49,837 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:51,600 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:56,600 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:57:56,941 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:56,942 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:57:56,942 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:57:56,943 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:57:56,944 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:57:56,944 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 6 < 516. Dropping entry: {'loss/epoch': 0.1204388655321456, '_timestamp': 1700215076.9417944}.
+2023-11-17 09:57:57,316 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:57:57,842 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:57:59,843 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:58:01,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:58:01,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:58:01,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:01,844 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:58:02,845 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:58:04,846 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:58:06,009 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:06,010 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:06,011 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:06,011 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:06,847 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:58:06,847 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:07,609 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:08,668 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:08,669 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:08,670 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:08,670 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:08,848 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 09:58:08,848 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:12,671 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:15,338 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:15,340 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:15,340 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:15,340 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:15,852 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:16,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:58:16,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:58:18,545 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:23,335 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:23,336 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:23,336 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:23,337 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:23,856 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:24,337 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:27,317 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:58:30,318 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:30,417 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:30,418 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:30,418 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:30,419 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:30,859 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:31,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:58:31,487 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:58:35,597 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:38,816 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:38,818 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:38,818 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:38,818 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:38,864 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:40,819 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:45,820 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:46,335 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:46,337 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:46,337 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:46,338 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:46,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:58:46,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:58:46,868 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:51,598 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:52,705 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:52,706 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:52,706 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:52,707 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:52,871 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:58:56,708 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:58:57,318 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:58:58,995 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:58:58,996 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:58:58,996 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:58:58,997 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:58:59,880 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:01,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:59:01,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:59:02,546 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:05,592 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:05,594 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:05,594 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:05,594 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:05,877 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:07,595 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:11,916 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:11,917 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:11,917 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:11,918 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:12,884 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:12,918 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:16,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:59:16,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:59:18,467 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:18,469 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:18,469 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:18,469 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:18,470 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:18,887 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:23,471 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:23,616 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:23,617 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:23,617 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:23,618 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:23,890 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:27,319 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:59:29,319 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:30,622 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:30,624 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:30,624 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:30,624 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:30,894 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:31,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:59:31,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:59:34,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:36,944 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:36,946 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:36,946 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:36,947 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:37,898 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:39,947 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:44,308 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:44,309 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:44,309 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:44,310 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:44,902 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:45,311 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:46,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 09:59:46,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 09:59:50,625 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:50,669 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:50,670 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:50,671 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:50,671 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:50,906 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 09:59:55,672 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 09:59:57,040 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 09:59:57,041 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 09:59:57,041 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 09:59:57,042 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 09:59:57,319 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 09:59:57,910 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:01,321 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:01,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:00:01,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:00:03,643 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:03,645 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:03,645 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:03,646 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:03,913 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:06,647 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:11,647 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:11,687 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:11,688 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:11,689 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:11,689 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:11,918 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:16,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:00:16,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:00:17,625 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:18,711 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:18,712 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:18,712 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:18,713 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:18,926 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:22,714 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:24,840 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:24,841 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:24,841 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:24,842 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:24,929 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:27,320 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:00:28,321 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:31,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:00:31,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:00:33,313 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:33,314 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:33,314 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:33,315 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:33,934 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:34,315 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:39,316 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:39,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:39,663 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:39,663 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:39,664 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:39,938 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:44,665 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:46,192 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:46,193 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:46,193 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:46,193 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:46,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:00:46,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:00:46,941 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:50,595 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:54,411 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:00:54,412 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:00:54,413 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:00:54,413 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:00:54,946 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:00:56,414 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:00:57,321 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:01:01,447 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:01,448 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:01,448 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:01,449 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:01,449 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:01,488 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:01:01,488 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:01:01,949 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:06,555 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:08,695 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:08,696 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:08,696 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:08,697 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:08,953 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:11,698 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:16,044 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:16,046 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:16,046 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:16,047 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:16,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:01:16,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:01:16,958 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:17,606 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:22,607 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:23,966 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:23,967 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:23,967 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:23,968 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:24,962 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:27,322 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:01:28,322 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:31,359 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:31,360 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:31,361 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:31,361 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:31,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:01:31,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:01:31,966 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:33,595 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:37,899 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:37,900 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:37,900 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:37,901 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:37,969 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:38,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:40,422 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:40,423 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:40,423 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:40,424 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:40,972 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:44,425 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:46,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:01:46,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:01:47,084 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:47,086 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:47,086 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:47,087 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:47,976 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:50,087 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:53,814 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:53,815 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:53,815 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:53,816 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:53,979 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:55,817 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:01:56,716 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:01:56,717 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:01:56,717 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:01:56,718 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:01:56,981 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:01:57,323 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:02:01,324 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:01,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:02:01,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:02:01,846 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:01,848 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:01,848 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:01,849 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:01,984 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:06,850 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:08,850 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:08,852 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:08,852 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:08,854 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:08,987 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:11,855 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:15,557 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:15,558 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:15,559 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:15,559 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:15,993 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:16,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:02:16,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:02:17,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:21,982 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:21,983 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:21,984 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:21,984 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:21,999 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:22,985 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:27,323 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:02:28,324 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:29,714 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:29,715 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:29,715 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:29,716 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:30,003 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:31,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:02:31,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:02:33,593 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:36,038 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:36,039 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:36,040 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:36,040 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:37,007 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:39,041 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:43,192 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:43,193 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:43,194 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:43,194 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:44,011 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:44,195 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:46,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:02:46,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:02:49,524 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:49,589 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:49,590 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:49,590 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:49,591 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:50,014 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:02:54,593 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:02:57,324 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:02:57,564 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:02:57,565 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:02:57,566 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:02:57,566 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:02:58,019 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:00,567 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:01,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:03:01,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:03:04,273 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:04,275 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:04,275 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:04,276 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:05,023 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:06,276 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:10,918 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:10,919 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:10,919 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:10,920 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:11,026 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:11,921 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:16,489 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:03:16,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:03:17,255 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:17,257 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:17,257 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:17,263 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:17,264 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:18,030 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:22,265 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:24,401 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:24,403 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:24,403 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:24,403 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:25,033 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:26,967 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:26,969 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:26,969 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:26,969 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:27,034 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:27,325 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:03:27,325 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:31,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:03:31,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:03:32,604 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:34,009 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:34,011 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:34,011 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:34,011 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:34,038 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:38,013 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:41,081 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:41,082 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:41,082 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:41,083 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:42,043 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:43,084 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:46,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:03:46,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:03:47,790 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:47,791 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:47,792 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:47,792 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:48,046 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:48,793 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:53,794 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:03:55,292 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:03:55,294 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:03:55,294 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:03:55,294 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:03:56,050 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:03:57,326 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:03:59,326 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:01,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:04:01,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:04:01,665 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:01,666 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:01,666 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:01,667 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:02,054 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:04,668 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:06,100 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:06,101 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:06,102 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:06,102 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:07,057 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:10,103 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:13,510 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:13,512 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:13,512 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:13,513 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:14,060 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:15,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:16,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:04:16,490 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:04:19,897 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:19,898 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:19,898 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:19,899 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:20,063 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:20,899 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:25,900 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:26,335 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:26,336 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:26,336 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:26,337 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:27,067 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:27,326 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:04:31,327 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:31,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:04:31,491 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:04:33,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:33,754 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:33,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:33,754 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:34,071 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:36,755 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:41,756 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:42,216 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:42,217 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:42,217 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:42,218 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:43,076 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:46,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:04:46,491 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:04:47,606 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:48,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:48,755 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:48,755 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:48,755 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:49,079 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:52,756 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:04:55,492 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:04:55,493 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:04:55,493 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:04:55,494 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:04:56,082 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:04:57,327 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:04:58,328 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:01,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:05:01,491 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:05:01,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:01,885 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:01,886 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:01,886 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:02,085 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:03,887 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:08,251 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:08,252 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:08,253 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:08,253 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:09,090 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:09,254 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:14,255 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:14,983 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:14,984 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:14,985 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:14,985 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:15,093 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:16,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:05:16,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:05:19,874 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:21,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:21,514 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:21,514 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:21,515 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:22,101 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:25,516 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:27,328 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:05:28,571 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:28,572 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:28,573 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:28,573 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:29,105 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:30,574 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:31,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:05:31,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:05:35,087 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:35,088 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:35,088 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:35,089 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:35,108 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:36,089 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:41,090 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:43,175 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:43,176 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:43,176 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:43,177 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:44,112 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:46,178 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:46,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:05:46,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:05:47,599 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:47,600 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:47,600 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:47,601 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:48,114 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:51,602 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:54,141 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:05:54,142 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:05:54,143 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:05:54,143 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:05:55,118 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:05:57,144 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:05:57,332 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:06:01,522 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:01,523 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:01,524 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:01,524 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:01,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:06:01,752 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:06:02,122 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:02,859 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:07,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:07,901 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:07,902 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:07,903 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:07,903 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:08,125 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:12,904 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:14,270 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:14,271 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:14,271 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:14,272 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:15,129 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:16,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:06:16,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:06:18,825 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:20,962 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:20,963 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:20,963 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:20,964 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:21,132 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:23,965 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:27,330 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:06:28,014 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:28,015 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:28,016 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:28,018 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:28,138 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:29,019 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:31,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:06:31,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:06:32,573 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:32,574 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:32,575 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:32,575 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:33,141 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:34,576 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:39,152 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:39,153 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:39,153 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:39,154 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:40,144 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:40,155 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:45,155 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:46,752 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:06:46,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:06:47,978 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:47,979 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:47,980 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:47,980 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:48,149 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:50,981 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:54,341 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:06:54,342 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:06:54,343 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:06:54,343 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:06:55,152 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:06:56,344 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:06:57,330 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:07:00,881 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:07:00,882 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:07:00,882 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:07:00,885 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:07:01,156 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:07:01,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:07:01,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:07:01,803 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:06,804 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:08,282 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:07:08,283 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:07:08,283 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:07:08,284 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:07:09,160 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:07:12,285 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:16,164 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:16,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:07:16,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:07:17,167 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:17,656 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:19,168 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:21,170 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:23,164 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:23,170 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:25,172 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:27,173 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:27,331 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:07:28,826 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:29,174 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:31,175 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:31,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:07:31,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:07:33,176 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:34,490 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:35,177 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:37,178 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:39,179 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:40,145 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:41,180 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:43,181 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:45,182 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:45,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:46,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:07:46,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:07:47,183 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:49,184 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:50,294 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 10:07:50,294 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_0_599_f33025ee89c6383b142f.png with policy now
+2023-11-17 10:07:50,295 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:07:50,296 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:07:50,297 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:07:50,297 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:07:50,986 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/lz0jna6c-media/images/sample_0_599_f33025ee89c6383b142f.png
+2023-11-17 10:07:51,185 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:07:51,186 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_599_f33025ee89c6383b142f.png
+2023-11-17 10:07:51,186 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 10:07:51,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:51,574 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/om6wmx6x-media/images/sample_0_599_f33025ee89c6383b142f.png
+2023-11-17 10:07:52,186 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:53,187 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:55,188 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:57,114 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:07:57,189 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:07:57,332 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:07:59,190 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:01,191 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:01,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:08:01,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:08:02,845 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:03,192 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:05,194 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:07,195 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:08,378 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:09,196 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:11,197 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:13,198 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:14,025 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:15,199 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:16,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:08:16,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:08:17,200 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:19,201 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:19,670 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:21,204 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:23,205 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:24,830 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 10:08:24,830 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_1_600_bf1a8515a6b246f936e2.png with policy now
+2023-11-17 10:08:24,835 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:24,835 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:24,837 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:08:24,837 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:08:24,837 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:08:24,856 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:24,856 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 600 < 601. Dropping entry: {'loss/current': 0.10467426478862762, 'loss/average': 0.12930890636756842, 'lr/unet': 4e-05, '_timestamp': 1700215704.8563807}.
+2023-11-17 10:08:25,052 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/4xvgmbe6-media/images/sample_1_600_bf1a8515a6b246f936e2.png
+2023-11-17 10:08:25,206 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:08:25,206 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_600_bf1a8515a6b246f936e2.png
+2023-11-17 10:08:25,207 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 10:08:25,561 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/nz5tk2k9-media/images/sample_1_600_bf1a8515a6b246f936e2.png
+2023-11-17 10:08:27,333 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:08:30,334 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:31,539 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:31,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:08:31,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:08:35,869 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:38,904 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:38,906 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:08:38,906 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:08:38,908 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:08:38,910 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:38,910 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 7 < 602. Dropping entry: {'loss/epoch': 0.12913682567887008, '_timestamp': 1700215718.9095914}.
+2023-11-17 10:08:39,219 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:08:40,214 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:40,912 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:42,215 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:45,218 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:46,093 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:46,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:08:46,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:08:47,219 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:48,053 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:48,054 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:08:48,054 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:08:48,054 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:08:48,220 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:08:49,220 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:51,222 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:08:51,267 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:55,205 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:08:55,206 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:08:55,206 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:08:55,207 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:08:55,224 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:08:57,207 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:08:57,333 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:09:01,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:09:01,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:09:01,912 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:01,913 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:01,914 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:01,914 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:02,228 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:02,915 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:07,916 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:08,293 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:08,295 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:08,295 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:08,295 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:09,232 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:13,297 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:14,650 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:14,652 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:14,652 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:14,655 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:15,236 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:16,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:09:16,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:09:18,883 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:21,424 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:21,425 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:21,425 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:21,426 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:22,240 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:24,427 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:27,334 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:09:27,757 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:27,758 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:27,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:27,759 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:28,243 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:29,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:31,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:09:31,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:09:34,100 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:34,102 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:34,102 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:34,103 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:34,247 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:35,103 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:40,104 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:40,618 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:40,620 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:40,620 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:40,621 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:41,251 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:45,622 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:46,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:09:46,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:09:47,327 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:47,328 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:47,329 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:47,331 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:48,255 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:49,880 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:49,881 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:49,882 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:49,882 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:50,256 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:50,883 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:55,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:09:56,867 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:09:56,868 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:09:56,868 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:09:56,869 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:09:57,261 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:09:57,335 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:10:01,336 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:01,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:10:01,754 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:10:04,730 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:04,732 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:04,732 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:04,732 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:05,266 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:06,733 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:09,415 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:09,416 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:09,416 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:09,417 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:10,269 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:12,418 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:15,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:15,750 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:15,750 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:15,750 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:16,272 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:16,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:10:16,755 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:10:17,479 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:17,480 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:17,480 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:17,483 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:17,483 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:18,273 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:22,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:24,669 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:24,670 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:24,670 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:24,671 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:25,277 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:27,336 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:10:28,336 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:31,415 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:31,416 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:31,417 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:31,417 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:31,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:10:31,755 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:10:32,281 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:33,883 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:37,813 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:37,815 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:37,815 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:37,816 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:38,285 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:39,816 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:44,817 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:44,982 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:44,983 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:44,983 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:44,984 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:45,289 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:46,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:10:46,821 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:10:49,921 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:52,050 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:10:52,051 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:10:52,052 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:10:52,054 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:10:52,293 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:10:55,055 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:10:57,336 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:11:00,029 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:00,031 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:00,031 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:00,032 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:00,297 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:01,033 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:01,820 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:11:01,821 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:11:06,593 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:06,594 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:06,594 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:06,595 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:06,595 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:07,301 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:11,596 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:13,337 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:13,338 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:13,339 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:13,339 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:14,305 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:16,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:11:16,821 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:11:16,946 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:18,133 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:18,134 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:18,135 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:18,135 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:18,308 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:22,136 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:25,130 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:25,131 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:25,131 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:25,134 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:25,312 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:27,337 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:11:27,338 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:31,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:11:31,822 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:11:32,952 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:34,064 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:34,065 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:34,065 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:34,066 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:34,317 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:38,067 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:40,465 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:40,466 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:40,466 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:40,467 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:41,321 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:43,468 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:46,783 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:46,784 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:46,784 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:46,784 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:46,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:11:46,821 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:11:47,324 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:48,943 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:53,943 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:11:55,129 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:11:55,130 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:11:55,131 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:11:55,131 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:11:55,329 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:11:57,338 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:11:59,339 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:01,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:12:01,822 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:12:02,197 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:02,198 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:02,198 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:02,199 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:02,333 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:05,200 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:09,198 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:09,199 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:09,199 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:09,200 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:09,337 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:10,201 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:15,202 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:15,922 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:15,923 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:15,923 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:15,924 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:16,341 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:16,821 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:12:16,822 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:12:20,891 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:23,917 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:23,918 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:23,918 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:23,919 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:24,345 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:25,920 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:27,339 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:12:29,855 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:29,857 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:29,857 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:29,858 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:30,349 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:31,859 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:32,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:12:32,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:12:37,307 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:38,692 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:38,694 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:38,694 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:38,694 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:39,354 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:42,696 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:43,624 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:43,626 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:43,626 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:43,626 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:44,357 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:47,246 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:12:47,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:12:48,309 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:50,694 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:50,695 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:50,695 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:50,696 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:51,361 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:53,697 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:12:57,238 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:12:57,239 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:12:57,240 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:12:57,240 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:12:57,339 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:12:57,364 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:12:59,340 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:02,247 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:13:02,248 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:13:02,713 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:02,714 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:02,715 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:02,715 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:03,367 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:04,716 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:09,060 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:09,061 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:09,062 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:09,062 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:09,370 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:10,063 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:15,067 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:15,457 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:15,458 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:15,459 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:15,459 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:16,374 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:17,247 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:13:17,247 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:13:20,327 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:21,777 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:21,778 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:21,778 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:21,779 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:22,378 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:24,833 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:24,834 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:24,834 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:24,835 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:25,379 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:25,835 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:27,340 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:13:31,222 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:31,224 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:31,224 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:31,225 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:31,225 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:31,383 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:32,247 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:13:32,248 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:13:36,353 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:38,415 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:38,416 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:38,416 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:38,417 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:39,387 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:41,418 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:46,278 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:46,279 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:46,280 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:46,280 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:46,391 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:47,248 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:47,248 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:13:47,249 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:13:52,322 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:13:54,758 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:13:54,760 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:13:54,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:13:54,762 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:13:55,396 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:13:57,341 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:13:57,344 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:02,248 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:14:02,248 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:14:02,368 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:02,877 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:02,878 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:02,878 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:02,879 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:03,401 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:07,880 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:10,307 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:10,308 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:10,308 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:10,309 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:10,405 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:13,309 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:17,320 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:14:17,320 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:14:17,979 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:17,980 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:17,981 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:17,981 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:18,409 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:18,982 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:23,983 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:26,310 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:26,311 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:26,312 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:26,312 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:26,413 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:27,342 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:14:29,343 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:32,320 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:14:32,320 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:14:33,745 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:33,746 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:33,746 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:33,746 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:34,418 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:34,747 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:39,748 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:40,795 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:40,796 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:40,796 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:40,797 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:41,422 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:44,798 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:47,320 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:14:47,321 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:14:47,360 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:47,401 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:47,402 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:47,402 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:47,425 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:50,403 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:54,064 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:14:54,065 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:14:54,065 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:14:54,066 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:14:54,429 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:14:56,067 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:14:57,348 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:15:00,507 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:00,508 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:00,509 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:00,509 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:01,433 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:01,510 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:02,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:15:02,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:15:06,876 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:08,793 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:08,794 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:08,794 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:08,795 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:09,437 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:12,795 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:16,247 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:16,248 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:16,248 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:16,249 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:16,441 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:17,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:15:17,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:15:17,904 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:22,905 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:23,262 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:23,263 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:23,264 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:23,264 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:23,445 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:27,349 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:15:28,350 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:29,648 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:29,650 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:29,650 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:29,650 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:30,449 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:32,758 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:15:32,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:15:33,896 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:37,031 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:37,033 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:37,033 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:37,033 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:37,453 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:39,034 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:44,035 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:44,981 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:44,983 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:44,983 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:44,984 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:45,458 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:47,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:15:47,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:15:49,823 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:52,006 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:52,007 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:52,008 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:52,008 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:52,462 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:15:55,009 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:15:57,350 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:15:59,418 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:15:59,420 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:15:59,420 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:15:59,421 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:15:59,466 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:00,421 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:02,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:16:02,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:16:05,783 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:05,784 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:05,785 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:05,785 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:05,786 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:06,470 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:10,787 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:12,120 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:12,122 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:12,122 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:12,123 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:12,474 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:16,124 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:17,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:16:17,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:16:18,769 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:18,770 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:18,770 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:18,771 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:19,478 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:21,772 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:25,477 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:25,479 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:25,479 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:25,479 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:25,482 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:27,351 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:16:27,353 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:31,853 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:31,855 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:31,855 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:31,855 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:32,486 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:32,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:16:32,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:16:32,910 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:37,912 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:38,430 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:38,431 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:38,432 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:38,432 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:38,491 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:43,433 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:45,159 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:45,161 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:45,161 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:45,161 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:45,495 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:47,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:16:47,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:16:48,979 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:52,189 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:52,190 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:52,191 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:52,191 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:52,499 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:16:54,192 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:57,351 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:16:59,352 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:16:59,361 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:16:59,362 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:16:59,362 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:16:59,363 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:16:59,503 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:01,966 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:01,967 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:01,968 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:01,968 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:02,505 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:02,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:17:02,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:17:04,903 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:08,744 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:08,745 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:08,745 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:08,746 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:09,509 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:10,746 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:15,094 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:15,096 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:15,096 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:15,097 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:15,513 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:16,097 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:17,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:17:17,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:17:21,769 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:21,770 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:21,770 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:21,771 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:21,771 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:22,517 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:26,772 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:27,352 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:17:28,495 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:28,496 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:28,496 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:28,497 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:28,520 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:32,498 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:32,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:17:32,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:17:34,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:34,861 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:34,861 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:34,862 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:35,524 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:37,863 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:41,431 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:41,432 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:41,432 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:41,432 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:41,528 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:43,433 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:47,744 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:47,745 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:47,745 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:47,746 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:47,759 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:17:47,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:17:48,531 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:48,797 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:53,798 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:17:54,118 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:54,120 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:54,120 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:54,121 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:54,535 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:57,353 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:17:58,519 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:17:58,520 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:17:58,521 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:17:58,521 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:17:58,537 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:17:59,522 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:02,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:18:02,761 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:18:04,885 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:05,686 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:05,687 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:05,687 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:05,688 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:06,545 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:10,689 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:13,114 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:13,115 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:13,116 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:13,116 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:13,117 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:13,118 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 8 < 688. Dropping entry: {'loss/epoch': 0.11307112428597933, '_timestamp': 1700216293.117459}.
+2023-11-17 10:18:13,549 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:14,550 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:18:16,121 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:16,551 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:18:17,551 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:18:17,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:18:17,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:18:19,552 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:18:21,219 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:21,553 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:18:23,157 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:23,159 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:23,159 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:23,160 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:23,554 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:18:23,555 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:27,161 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:27,353 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:18:30,550 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:30,551 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:30,552 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:30,552 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:30,558 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:32,553 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:32,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:18:32,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:18:36,899 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:36,900 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:36,901 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:36,901 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:37,562 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:37,902 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:42,848 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:42,850 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:42,850 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:42,851 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:43,565 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:43,851 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:44,589 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:44,590 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:44,591 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:44,591 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:45,567 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:47,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:18:47,760 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:18:48,884 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:52,008 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:52,009 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:52,010 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:52,010 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:18:52,570 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:18:54,012 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:57,354 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:18:59,355 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:18:59,981 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:18:59,982 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:18:59,983 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:18:59,983 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:00,575 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:02,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:19:02,761 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:19:04,824 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:06,352 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:06,353 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:06,353 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:06,354 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:06,578 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:10,355 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:13,406 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:13,407 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:13,407 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:13,408 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:13,582 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:15,409 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:17,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:19:17,761 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:19:20,516 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:20,517 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:20,518 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:20,518 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:20,519 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:20,586 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:25,522 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:27,355 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:19:28,480 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:28,481 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:28,481 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:28,482 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:28,590 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:31,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:31,022 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:31,023 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:31,023 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:31,024 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:31,592 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:32,809 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:19:32,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:19:36,880 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:37,774 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:37,775 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:37,775 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:37,776 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:38,596 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:42,778 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:44,134 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:44,135 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:44,135 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:44,136 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:44,599 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:47,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:19:47,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:19:47,923 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:50,453 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:50,454 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:50,454 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:50,455 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:50,603 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:53,456 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:19:57,165 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:19:57,167 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:19:57,167 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:19:57,168 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:19:57,356 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:19:57,607 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:19:59,357 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:02,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:20:02,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:20:03,733 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:03,735 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:03,735 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:03,736 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:04,610 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:04,736 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:09,737 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:10,456 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:10,457 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:10,457 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:10,458 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:10,614 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:15,459 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:17,400 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:17,402 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:17,402 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:17,402 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:17,618 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:17,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:20:17,809 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:20:20,886 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:24,132 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:24,133 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:24,134 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:24,134 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:24,621 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:26,135 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:27,357 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:20:31,158 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:31,159 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:31,160 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:31,160 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:31,160 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:31,625 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:32,928 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:20:32,928 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:20:37,003 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:37,633 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:37,634 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:37,634 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:37,635 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:38,630 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:42,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:44,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:44,383 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:44,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:44,384 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:44,633 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:47,928 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:47,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:20:47,929 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:20:51,420 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:51,421 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:51,422 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:51,422 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:51,637 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:53,423 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:20:57,357 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:20:57,751 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:20:57,753 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:20:57,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:20:57,753 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:20:58,645 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:20:58,754 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:02,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:21:02,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:21:04,075 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:05,187 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:05,188 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:05,188 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:05,189 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:05,649 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:09,190 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:11,567 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:11,568 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:11,569 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:11,569 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:11,652 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:14,570 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:17,870 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:17,871 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:17,872 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:17,872 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:17,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:21:17,929 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:21:18,662 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:20,021 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:25,022 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:25,069 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:25,070 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:25,070 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:25,071 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:25,666 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:27,358 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:21:30,359 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:32,508 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:32,509 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:32,510 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:32,510 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:32,669 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:32,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:21:32,929 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:21:35,997 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:40,029 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:40,031 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:40,031 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:40,032 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:40,674 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:41,033 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:44,980 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:44,981 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:44,981 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:44,982 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:45,677 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:46,983 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:47,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:21:47,929 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:21:51,716 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:51,717 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:51,718 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:51,718 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:52,680 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:21:52,719 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:57,359 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:21:58,360 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:21:58,739 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:21:58,740 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:21:58,740 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:21:58,741 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:21:59,687 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:02,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:22:02,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:22:03,992 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:05,299 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:05,301 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:05,301 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:05,301 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:05,691 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:09,303 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:12,026 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:12,027 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:12,028 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:12,028 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:12,695 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:15,029 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:17,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:22:17,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:22:19,076 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:19,077 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:19,077 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:19,078 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:19,699 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:20,079 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:25,080 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:27,184 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:27,185 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:27,186 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:27,186 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:27,360 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:22:27,703 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:30,360 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:32,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:22:32,929 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:22:33,946 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:33,947 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:33,947 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:33,948 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:34,707 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:35,949 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:40,950 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:42,476 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:42,478 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:42,478 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:42,479 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:42,712 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:46,480 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:47,930 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:22:47,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:22:50,656 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:50,657 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:50,657 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:50,658 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:50,716 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:22:51,658 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:56,659 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:22:57,360 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:22:57,417 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:22:57,418 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:22:57,418 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:22:57,419 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:22:57,720 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:02,420 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:02,929 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:23:02,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:23:03,357 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:03,358 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:03,358 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:03,359 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:03,725 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:08,360 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:09,812 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:09,813 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:09,814 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:09,814 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:10,729 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:13,815 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:17,221 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:17,223 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:17,223 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:17,223 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:17,733 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:17,930 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:23:17,930 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:23:19,059 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:23,625 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:23,626 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:23,627 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:23,627 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:23,736 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:24,628 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:27,361 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:23:29,998 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:29,999 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:29,999 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:30,000 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:30,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:30,740 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:32,930 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:23:32,931 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:23:35,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:37,434 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:37,436 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:37,436 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:37,436 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:37,744 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:40,437 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:43,814 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:43,815 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:43,815 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:43,816 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:44,748 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:45,816 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:47,930 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:23:47,931 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:23:51,013 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:51,775 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:51,776 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:51,776 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:51,777 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:52,753 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:23:56,778 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:23:57,362 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:23:58,487 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:23:58,489 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:23:58,489 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:23:58,490 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:23:58,756 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:02,491 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:02,931 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:24:02,931 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:24:05,110 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:05,111 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:05,111 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:05,112 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:05,760 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:08,113 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:12,158 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:12,159 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:12,159 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:12,160 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:12,766 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:13,161 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:18,005 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:24:18,006 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:24:18,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:18,637 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:18,637 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:18,638 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:18,638 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:18,770 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:23,639 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:24,978 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:24,979 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:24,980 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:24,980 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:25,774 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:27,363 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:24:29,364 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:31,346 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:31,347 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:31,348 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:31,348 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:31,777 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:33,112 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:24:33,112 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:24:35,210 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:37,874 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:37,876 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:37,876 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:37,877 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:38,781 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:40,878 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:45,327 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:45,328 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:45,328 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:45,329 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:45,784 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:46,330 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:48,112 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:24:48,113 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:24:51,679 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:51,681 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:51,681 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:51,682 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:51,682 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:51,788 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:24:56,683 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:24:57,363 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:24:58,894 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:24:58,895 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:24:58,895 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:24:58,896 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:24:59,792 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:25:01,897 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:03,112 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:25:03,113 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:25:05,645 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:25:05,647 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:25:05,647 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:25:05,648 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:25:05,795 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:25:07,648 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:10,798 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:11,799 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:12,673 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:13,801 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:15,802 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:17,804 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:18,112 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:18,113 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:25:18,113 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:25:19,807 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:21,808 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:23,809 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:23,853 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:25,811 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:27,364 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:25:27,812 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:29,507 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:29,813 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:31,814 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:33,113 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:25:33,113 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:25:33,815 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:35,209 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:35,816 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:37,817 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:39,818 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:40,830 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:41,820 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:43,821 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:45,275 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 10:25:45,276 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_0_749_b4584a4b456499262774.png with policy now
+2023-11-17 10:25:45,275 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:25:45,278 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:25:45,279 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:25:45,284 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:25:45,822 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:25:45,822 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_749_b4584a4b456499262774.png
+2023-11-17 10:25:45,823 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 10:25:45,848 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/oa7hzqxp-media/images/sample_0_749_b4584a4b456499262774.png
+2023-11-17 10:25:46,205 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/tvb9k8am-media/images/sample_0_749_b4584a4b456499262774.png
+2023-11-17 10:25:46,528 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:46,822 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:47,823 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:48,113 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:25:48,113 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:25:49,824 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:51,825 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:52,160 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:53,826 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:55,828 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:57,365 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:25:57,367 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:25:57,829 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:25:59,830 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:01,831 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:02,410 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:03,113 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:26:03,113 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:26:03,832 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:05,834 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:07,835 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:07,908 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:09,836 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:11,837 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:13,537 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:13,838 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:15,839 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:17,840 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:26:18,113 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:26:18,114 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:26:19,207 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:19,804 DEBUG   SenderThread:11593 [sender.py:send():379] send: files
+2023-11-17 10:26:19,804 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file media/images/sample_1_750_a3327d6a96d67d67e385.png with policy now
+2023-11-17 10:26:19,841 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_created():272] file/dir created: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_750_a3327d6a96d67d67e385.png
+2023-11-17 10:26:19,842 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images
+2023-11-17 10:26:19,845 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:19,845 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:19,845 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 750 < 751. Dropping entry: {'loss/current': 0.13325612246990204, 'loss/average': 0.12559634666121022, 'lr/unet': 4e-05, '_timestamp': 1700216779.822489}.
+2023-11-17 10:26:19,846 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:26:19,846 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:26:19,847 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:26:20,018 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/xbs5o4pd-media/images/sample_1_750_a3327d6a96d67d67e385.png
+2023-11-17 10:26:20,322 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /tmp/tmpx0jzx6ttwandb/qqgjxgay-media/images/sample_1_750_a3327d6a96d67d67e385.png
+2023-11-17 10:26:20,842 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:26:24,848 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:26,147 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:27,365 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:26:30,366 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:31,406 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:31,408 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:26:31,408 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:26:31,409 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:26:31,850 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:26:33,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:26:33,382 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:26:35,433 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:38,758 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:38,759 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:26:38,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:26:38,760 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:26:38,854 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:26:40,761 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:41,528 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:41,529 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:26:41,530 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:26:41,530 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:26:41,856 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:26:46,531 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:48,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:26:48,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:26:48,482 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:48,483 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:26:48,483 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:26:48,483 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:26:48,860 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:26:52,484 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:26:54,939 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:26:54,940 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:26:54,940 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:26:54,941 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:26:55,864 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:26:57,366 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:26:58,367 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:02,299 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:02,300 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:02,301 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:02,301 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:02,867 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:03,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:27:03,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:27:03,453 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:08,454 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:09,315 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:09,316 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:09,316 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:09,317 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:09,871 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:14,318 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:17,006 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:17,007 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:17,008 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:17,008 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:17,876 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:18,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:27:18,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:27:19,421 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:24,422 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:24,441 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:24,443 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:24,443 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:24,443 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:24,880 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:27,367 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:27:30,368 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:30,820 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:30,822 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:30,822 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:30,822 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:30,883 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:33,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:27:33,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:27:35,419 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:35,420 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:35,420 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:35,421 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:35,422 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:35,886 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:40,422 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:43,297 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:43,298 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:43,298 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:43,299 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:43,891 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:46,300 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:47,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:47,637 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:47,637 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:47,638 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:47,893 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:48,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:27:48,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:27:51,456 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:56,108 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:27:56,109 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:27:56,109 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:27:56,110 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:27:56,901 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:27:57,111 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:27:57,368 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:28:02,369 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:02,850 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:02,851 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:02,851 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:02,852 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:02,904 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:03,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:28:03,383 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:28:07,501 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:09,907 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:09,909 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:09,909 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:09,910 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:10,909 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:12,910 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:17,872 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:17,873 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:17,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:17,874 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:17,913 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:18,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:28:18,384 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:28:18,531 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:23,531 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:24,618 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:24,620 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:24,620 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:24,620 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:24,917 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:27,368 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:28:29,369 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:30,989 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:30,990 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:30,990 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:30,991 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:31,921 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:33,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:28:33,384 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:28:34,473 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:37,387 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:37,389 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:37,389 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:37,389 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:37,926 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:40,390 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:43,947 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:43,948 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:43,948 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:43,949 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:44,930 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:45,950 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:48,383 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:28:48,384 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:28:51,494 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:28:52,402 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:28:52,403 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:28:52,404 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:28:52,404 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:28:52,935 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:28:57,369 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:28:57,371 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:00,340 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:00,341 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:00,342 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:00,342 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:00,343 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:00,343 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 9 < 774. Dropping entry: {'loss/epoch': 0.12433343515244075, '_timestamp': 1700216940.3406076}.
+2023-11-17 10:29:00,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:01,940 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:29:03,089 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:03,385 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:29:03,385 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:29:03,942 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:29:05,495 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:05,497 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:05,497 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:05,497 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:05,943 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:29:05,943 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:07,944 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:29:08,119 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:09,945 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:29:11,947 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:29:12,250 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:12,251 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:12,251 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:12,252 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:12,947 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:13,252 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:16,862 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:16,863 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:16,863 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:16,864 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:16,949 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:18,511 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:29:18,512 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:29:18,642 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:22,789 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:22,790 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:22,790 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:22,791 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:22,953 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:23,792 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:27,370 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:29:29,371 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:31,176 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:31,177 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:31,178 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:31,178 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:31,958 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:33,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:29:33,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:29:34,623 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:38,216 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:38,218 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:38,218 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:38,219 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:38,963 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:40,220 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:44,607 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:44,608 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:44,608 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:44,609 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:44,968 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:45,609 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:48,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:29:48,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:29:50,631 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:51,287 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:51,288 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:51,289 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:51,289 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:51,972 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:29:56,292 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:29:57,371 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:29:57,633 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:29:57,634 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:29:57,634 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:29:57,635 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:29:57,976 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:01,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:03,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:30:03,512 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:30:04,681 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:04,682 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:04,682 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:04,683 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:04,980 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:06,684 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:11,343 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:11,344 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:11,344 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:11,345 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:11,984 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:12,345 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:13,791 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:13,793 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:13,793 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:13,793 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:13,985 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:17,794 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:18,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:30:18,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:30:20,266 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:20,268 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:20,268 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:20,268 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:20,989 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:23,269 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:26,582 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:26,583 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:26,583 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:26,584 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:26,993 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:27,371 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:30:28,372 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:33,134 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:33,135 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:33,136 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:33,136 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:33,512 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:33,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:30:33,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:30:33,997 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:38,651 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:39,878 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:39,879 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:39,879 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:39,880 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:40,000 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:43,881 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:48,286 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:48,287 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:48,287 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:48,288 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:48,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:30:48,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:30:49,006 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:49,657 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:50,614 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:50,615 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:50,616 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:50,616 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:51,007 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:30:55,617 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:30:57,339 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:30:57,341 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:30:57,341 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:30:57,341 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:30:57,372 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:30:58,011 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:00,306 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:00,307 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:00,308 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:00,308 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:01,013 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:01,309 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:03,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:31:03,513 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:31:06,613 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:06,694 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:06,695 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:06,696 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:06,696 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:07,016 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:11,697 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:13,049 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:13,050 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:13,050 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:13,051 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:14,020 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:17,052 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:18,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:31:18,514 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:31:19,757 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:19,759 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:19,759 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:19,760 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:20,024 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:22,761 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:27,287 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:27,288 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:27,288 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:27,289 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:27,377 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:31:28,028 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:28,377 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:33,378 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:33,513 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:31:33,514 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:31:33,647 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:33,648 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:33,648 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:33,649 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:34,032 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:38,650 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:40,361 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:40,362 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:40,363 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:40,363 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:41,036 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:44,364 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:45,559 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:45,560 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:45,560 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:45,561 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:46,040 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:48,514 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:31:48,514 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:31:49,592 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:51,945 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:51,946 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:51,946 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:51,947 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:52,043 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:31:54,948 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:31:57,373 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:31:58,610 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:31:58,611 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:31:58,611 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:31:58,611 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:31:59,047 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:00,612 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:03,514 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:32:03,514 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:32:05,456 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:05,458 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:05,458 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:05,459 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:06,051 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:06,459 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:11,460 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:12,251 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:12,252 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:12,252 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:12,253 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:13,055 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:16,674 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:16,676 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:16,676 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:16,677 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:16,677 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:17,058 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:18,663 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:32:18,664 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:32:21,730 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:23,673 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:23,674 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:23,674 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:23,675 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:24,062 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:27,374 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:32:27,375 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:31,568 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:31,570 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:31,570 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:31,573 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:32,067 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:32,573 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:33,661 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:32:33,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:32:37,780 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:38,621 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:38,622 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:38,622 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:38,623 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:39,071 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:43,624 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:45,324 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:45,325 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:45,325 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:45,326 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:46,075 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:48,661 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:32:48,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:32:48,688 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:52,723 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:52,724 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:52,725 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:52,725 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:32:53,079 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:32:53,726 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:57,375 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:32:59,376 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:32:59,815 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:32:59,817 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:32:59,817 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:32:59,817 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:00,083 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:03,661 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:33:03,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:33:04,803 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:06,856 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:06,857 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:06,857 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:06,859 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:07,088 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:09,860 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:13,605 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:13,606 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:13,606 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:13,607 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:14,091 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:15,608 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:18,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:33:18,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:33:19,993 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:19,994 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:19,994 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:19,995 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:20,095 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:20,996 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:25,997 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:26,357 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:26,358 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:26,358 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:26,359 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:27,099 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:27,376 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:33:31,377 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:32,865 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:32,866 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:32,866 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:32,867 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:33,103 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:33,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:33:33,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:33:36,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:38,398 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:38,400 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:38,400 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:38,406 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:39,106 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:42,407 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:44,757 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:44,758 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:44,758 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:44,759 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:45,110 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:47,760 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:48,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:33:48,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:33:51,226 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:51,227 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:51,228 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:51,228 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:52,114 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:33:53,229 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:57,377 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:33:58,378 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:33:58,633 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:33:58,634 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:33:58,635 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:33:58,635 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:33:59,120 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:03,636 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:03,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:34:03,662 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:34:05,037 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:05,039 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:05,039 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:05,040 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:05,124 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:09,040 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:13,182 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:13,183 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:13,184 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:13,186 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:14,131 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:14,187 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:18,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:34:18,663 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:34:19,719 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:21,546 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:21,548 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:21,548 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:21,548 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:22,136 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:25,549 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:27,378 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:34:28,555 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:28,556 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:28,556 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:28,557 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:29,140 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:30,558 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:33,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:34:33,663 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:34:35,772 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:36,677 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:36,678 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:36,678 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:36,679 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:37,145 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:41,680 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:43,435 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:43,436 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:43,437 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:43,437 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:44,149 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:47,438 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:48,662 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:34:48,663 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:34:49,772 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:49,773 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:49,773 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:49,774 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:50,153 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:34:52,776 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:57,378 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:34:57,880 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:34:57,881 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:34:57,881 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:34:57,882 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:34:57,882 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:34:58,158 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:02,883 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:03,663 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:35:03,663 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:35:04,625 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:04,626 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:04,626 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:04,627 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:05,164 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:08,628 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:12,632 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:12,633 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:12,633 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:12,634 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:13,170 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:13,635 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:16,896 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:16,897 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:16,898 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:16,901 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:17,180 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:18,872 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:35:18,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:35:18,943 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:23,945 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:24,260 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:24,261 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:24,262 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:24,263 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:25,184 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:27,379 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:35:29,380 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:31,297 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:31,298 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:31,298 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:31,299 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:32,188 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:33,872 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:35:33,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:35:34,925 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:38,803 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:38,805 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:38,805 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:38,805 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:39,200 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:40,806 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:45,094 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:45,095 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:45,096 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:45,096 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:45,203 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:46,097 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:48,872 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:35:48,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:35:51,599 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:51,600 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:51,600 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:51,603 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:51,603 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:52,207 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:35:56,604 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:35:57,380 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:35:57,956 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:35:57,957 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:35:57,958 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:35:57,958 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:35:58,211 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:00,646 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:00,647 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:00,647 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:00,648 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:01,213 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:01,648 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:03,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:36:03,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:36:06,959 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:09,469 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:09,470 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:09,470 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:09,471 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:10,218 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:12,472 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:16,525 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:16,527 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:16,527 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:16,527 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:17,223 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:17,528 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:18,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:36:18,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:36:23,000 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:23,757 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:23,758 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:23,758 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:23,759 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:24,227 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:27,381 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:36:28,382 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:30,460 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:30,462 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:30,462 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:30,463 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:31,231 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:33,463 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:33,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:36:33,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:36:36,805 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:36,806 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:36,806 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:36,807 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:37,235 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:38,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:43,808 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:43,831 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:43,832 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:43,832 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:43,833 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:44,240 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:48,834 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:48,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:36:48,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:36:52,600 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:52,602 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:52,602 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:52,603 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:53,245 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:36:54,604 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:36:57,382 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:36:59,640 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:36:59,641 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:36:59,642 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:36:59,642 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:36:59,643 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:00,249 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:03,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:37:03,873 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:37:04,924 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:06,648 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:06,650 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:06,650 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:06,650 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:07,253 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:10,651 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:13,404 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:13,406 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:13,406 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:13,406 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:14,258 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:16,407 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:18,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:37:18,874 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:37:19,767 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:19,768 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:19,769 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:19,769 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:20,268 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:21,770 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:26,771 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:27,382 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:37:27,749 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:27,751 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:27,751 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:27,751 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:28,273 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:32,753 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:33,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:37:33,874 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:37:35,142 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:35,143 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:35,143 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:35,144 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:35,277 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:38,145 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:42,161 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:42,162 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:42,162 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:42,163 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:42,281 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:43,164 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:48,165 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:48,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:37:48,874 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:37:49,377 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:49,378 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:49,378 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:49,379 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:50,286 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:53,380 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:37:56,068 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:37:56,069 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:37:56,070 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:37:56,070 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:37:56,290 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:37:57,383 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:37:58,384 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:02,841 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:38:02,844 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:38:02,845 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:38:02,845 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:03,294 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:38:03,846 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:03,873 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:38:03,874 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:38:08,999 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:09,217 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:38:09,218 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:38:09,218 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:38:09,219 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:09,298 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:38:14,220 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:16,212 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:38:16,213 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:38:16,214 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:38:16,214 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:16,302 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:38:18,874 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: stop_status
+2023-11-17 10:38:18,874 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: stop_status
+2023-11-17 10:38:19,947 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:23,595 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:38:23,596 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:38:23,597 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:38:23,597 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:24,306 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:38:25,598 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:27,384 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:38:29,955 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:38:29,956 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:38:29,956 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:38:29,957 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:29,957 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: partial_history
+2023-11-17 10:38:29,957 WARNING HandlerThread:11593 [handler.py:handle_request_partial_history():553] Step 10 < 860. Dropping entry: {'loss/epoch': 0.13492779188984355, '_timestamp': 1700217509.955257}.
+2023-11-17 10:38:29,958 DEBUG   SenderThread:11593 [sender.py:send():379] send: telemetry
+2023-11-17 10:38:29,958 DEBUG   SenderThread:11593 [sender.py:send():379] send: exit
+2023-11-17 10:38:29,958 INFO    SenderThread:11593 [sender.py:send_exit():584] handling exit code: 0
+2023-11-17 10:38:29,959 INFO    SenderThread:11593 [sender.py:send_exit():586] handling runtime: 6092
+2023-11-17 10:38:29,959 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:29,959 INFO    SenderThread:11593 [sender.py:send_exit():592] send defer
+2023-11-17 10:38:29,959 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,959 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 0
+2023-11-17 10:38:29,959 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,960 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 0
+2023-11-17 10:38:29,960 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 1
+2023-11-17 10:38:29,960 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,960 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 1
+2023-11-17 10:38:29,960 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,960 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 1
+2023-11-17 10:38:29,960 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 2
+2023-11-17 10:38:29,960 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,960 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 2
+2023-11-17 10:38:29,960 INFO    HandlerThread:11593 [system_monitor.py:finish():190] Stopping system monitor
+2023-11-17 10:38:29,960 DEBUG   SystemMonitor:11593 [system_monitor.py:_start():166] Finished system metrics aggregation loop
+2023-11-17 10:38:29,960 DEBUG   SystemMonitor:11593 [system_monitor.py:_start():170] Publishing last batch of metrics
+2023-11-17 10:38:29,961 INFO    HandlerThread:11593 [interfaces.py:finish():202] Joined cpu monitor
+2023-11-17 10:38:29,961 INFO    HandlerThread:11593 [interfaces.py:finish():202] Joined disk monitor
+2023-11-17 10:38:29,969 INFO    HandlerThread:11593 [interfaces.py:finish():202] Joined gpu monitor
+2023-11-17 10:38:29,970 INFO    HandlerThread:11593 [interfaces.py:finish():202] Joined memory monitor
+2023-11-17 10:38:29,970 INFO    HandlerThread:11593 [interfaces.py:finish():202] Joined network monitor
+2023-11-17 10:38:29,970 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,970 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 2
+2023-11-17 10:38:29,970 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 3
+2023-11-17 10:38:29,970 DEBUG   SenderThread:11593 [sender.py:send():379] send: stats
+2023-11-17 10:38:29,971 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,971 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 3
+2023-11-17 10:38:29,971 DEBUG   SenderThread:11593 [sender.py:send():379] send: history
+2023-11-17 10:38:29,971 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: summary_record
+2023-11-17 10:38:29,972 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:29,972 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,972 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 3
+2023-11-17 10:38:29,972 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 4
+2023-11-17 10:38:29,972 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,972 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 4
+2023-11-17 10:38:29,973 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,973 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 4
+2023-11-17 10:38:29,973 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 5
+2023-11-17 10:38:29,973 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,973 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 5
+2023-11-17 10:38:29,973 DEBUG   SenderThread:11593 [sender.py:send():379] send: summary
+2023-11-17 10:38:29,974 INFO    SenderThread:11593 [sender.py:_save_file():1376] saving file wandb-summary.json with policy end
+2023-11-17 10:38:29,974 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,974 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 5
+2023-11-17 10:38:29,974 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 6
+2023-11-17 10:38:29,974 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:29,974 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 6
+2023-11-17 10:38:29,974 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:29,974 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 6
+2023-11-17 10:38:29,978 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: status_report
+2023-11-17 10:38:30,054 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 7
+2023-11-17 10:38:30,054 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:30,054 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 7
+2023-11-17 10:38:30,054 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:30,054 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 7
+2023-11-17 10:38:30,310 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:38:30,310 INFO    Thread-12 :11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/config.yaml
+2023-11-17 10:38:30,959 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: poll_exit
+2023-11-17 10:38:32,496 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 8
+2023-11-17 10:38:32,496 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: poll_exit
+2023-11-17 10:38:32,496 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:32,497 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 8
+2023-11-17 10:38:32,497 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:32,497 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 8
+2023-11-17 10:38:32,497 INFO    SenderThread:11593 [job_builder.py:build():280] Attempting to build job artifact
+2023-11-17 10:38:32,498 INFO    SenderThread:11593 [job_builder.py:_get_source_type():389] is repo sourced job
+2023-11-17 10:38:32,499 INFO    SenderThread:11593 [job_builder.py:build():363] adding wandb-job metadata file
+2023-11-17 10:38:32,504 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 9
+2023-11-17 10:38:32,505 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:32,505 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 9
+2023-11-17 10:38:32,505 DEBUG   SenderThread:11593 [sender.py:send():379] send: artifact
+2023-11-17 10:38:32,725 INFO    SenderThread:11593 [sender.py:send_artifact():1476] sent artifact job-https___github.com_qaneel_kohya-trainer_sdxl_train_network.py - {'id': 'QXJ0aWZhY3Q6NjQwNjQ0OTYz', 'digest': '1213857caaffe12535590390b82e9122', 'state': 'COMMITTED', 'aliases': [{'artifactCollectionName': 'job-https___github.com_qaneel_kohya-trainer_sdxl_train_network.py', 'alias': 'latest'}, {'artifactCollectionName': 'job-https___github.com_qaneel_kohya-trainer_sdxl_train_network.py', 'alias': 'v0'}], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjExNjM5NTUzOQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6NjQxMzE0Njcx', 'versionIndex': 1}}, 'version': 'v0'}
+2023-11-17 10:38:32,726 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:32,726 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 9
+2023-11-17 10:38:32,726 INFO    SenderThread:11593 [dir_watcher.py:finish():359] shutting down directory watcher
+2023-11-17 10:38:33,311 INFO    SenderThread:11593 [dir_watcher.py:_on_file_modified():289] file/dir modified: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:38:33,312 INFO    SenderThread:11593 [dir_watcher.py:finish():389] scan: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files
+2023-11-17 10:38:33,312 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/config.yaml config.yaml
+2023-11-17 10:38:33,312 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log output.log
+2023-11-17 10:38:33,315 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json wandb-summary.json
+2023-11-17 10:38:33,315 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-metadata.json wandb-metadata.json
+2023-11-17 10:38:33,316 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/requirements.txt requirements.txt
+2023-11-17 10:38:33,320 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_600_bf1a8515a6b246f936e2.png media/images/sample_1_600_bf1a8515a6b246f936e2.png
+2023-11-17 10:38:33,321 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_750_a3327d6a96d67d67e385.png media/images/sample_1_750_a3327d6a96d67d67e385.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_449_5d64336bece18f607be4.png media/images/sample_0_449_5d64336bece18f607be4.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_300_9c81ac6c681cfe260b5e.png media/images/sample_1_300_9c81ac6c681cfe260b5e.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_299_21ebb3119669773f9840.png media/images/sample_0_299_21ebb3119669773f9840.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_599_f33025ee89c6383b142f.png media/images/sample_0_599_f33025ee89c6383b142f.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_749_b4584a4b456499262774.png media/images/sample_0_749_b4584a4b456499262774.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_450_96ba8a2d2129a7a35156.png media/images/sample_1_450_96ba8a2d2129a7a35156.png
+2023-11-17 10:38:33,323 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_0_149_9acbaa99fb15c017cbb0.png media/images/sample_0_149_9acbaa99fb15c017cbb0.png
+2023-11-17 10:38:33,324 INFO    SenderThread:11593 [dir_watcher.py:finish():403] scan save: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/media/images/sample_1_150_b7c26fa9bbca9c3cddb4.png media/images/sample_1_150_b7c26fa9bbca9c3cddb4.png
+2023-11-17 10:38:33,324 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 10
+2023-11-17 10:38:33,326 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:33,326 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 10
+2023-11-17 10:38:33,327 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:33,328 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 10
+2023-11-17 10:38:33,328 INFO    SenderThread:11593 [file_pusher.py:finish():159] shutting down file pusher
+2023-11-17 10:38:33,957 INFO    wandb-upload_0:11593 [upload_job.py:push():131] Uploaded file /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/config.yaml
+2023-11-17 10:38:34,269 INFO    wandb-upload_2:11593 [upload_job.py:push():131] Uploaded file /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/wandb-summary.json
+2023-11-17 10:38:34,346 INFO    wandb-upload_1:11593 [upload_job.py:push():131] Uploaded file /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/output.log
+2023-11-17 10:38:34,365 INFO    wandb-upload_3:11593 [upload_job.py:push():131] Uploaded file /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/files/requirements.txt
+2023-11-17 10:38:34,566 INFO    Thread-11 (_thread_body):11593 [sender.py:transition_state():612] send defer: 11
+2023-11-17 10:38:34,566 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:34,567 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 11
+2023-11-17 10:38:34,567 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:34,567 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 11
+2023-11-17 10:38:34,567 INFO    SenderThread:11593 [file_pusher.py:join():164] waiting for file pusher
+2023-11-17 10:38:34,567 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 12
+2023-11-17 10:38:34,568 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:34,568 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 12
+2023-11-17 10:38:34,568 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:34,568 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 12
+2023-11-17 10:38:34,720 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 13
+2023-11-17 10:38:34,721 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:34,721 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 13
+2023-11-17 10:38:34,721 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:34,721 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 13
+2023-11-17 10:38:34,721 INFO    SenderThread:11593 [sender.py:transition_state():612] send defer: 14
+2023-11-17 10:38:34,721 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: defer
+2023-11-17 10:38:34,721 INFO    HandlerThread:11593 [handler.py:handle_request_defer():170] handle defer: 14
+2023-11-17 10:38:34,722 DEBUG   SenderThread:11593 [sender.py:send():379] send: final
+2023-11-17 10:38:34,722 DEBUG   SenderThread:11593 [sender.py:send():379] send: footer
+2023-11-17 10:38:34,722 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: defer
+2023-11-17 10:38:34,722 INFO    SenderThread:11593 [sender.py:send_request_defer():608] handle sender defer: 14
+2023-11-17 10:38:34,723 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: poll_exit
+2023-11-17 10:38:34,724 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: poll_exit
+2023-11-17 10:38:34,724 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: server_info
+2023-11-17 10:38:34,724 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: get_summary
+2023-11-17 10:38:34,725 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: sampled_history
+2023-11-17 10:38:34,726 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: job_info
+2023-11-17 10:38:34,726 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: server_info
+2023-11-17 10:38:34,753 DEBUG   SenderThread:11593 [sender.py:send_request():406] send_request: job_info
+2023-11-17 10:38:34,769 DEBUG   HandlerThread:11593 [handler.py:handle_request():144] handle_request: shutdown
+2023-11-17 10:38:34,769 INFO    HandlerThread:11593 [handler.py:finish():857] shutting down handler
+2023-11-17 10:38:35,726 INFO    WriterThread:11593 [datastore.py:close():294] close: /content/LoRA/logs/20231117085552/wandb/run-20231117_085656-hf8ykeyx/run-hf8ykeyx.wandb
+2023-11-17 10:38:35,753 INFO    SenderThread:11593 [sender.py:finish():1552] shutting down sender
+2023-11-17 10:38:35,753 INFO    SenderThread:11593 [file_pusher.py:finish():159] shutting down file pusher
+2023-11-17 10:38:35,753 INFO    SenderThread:11593 [file_pusher.py:join():164] waiting for file pusher