diff --git "a/wandb/run-20230608_122450-vrqnfbac/logs/debug-internal.log" "b/wandb/run-20230608_122450-vrqnfbac/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20230608_122450-vrqnfbac/logs/debug-internal.log" @@ -0,0 +1,1458 @@ +2023-06-08 12:24:50,643 INFO StreamThr :5514 [internal.py:wandb_internal():86] W&B internal server running at pid: 5514, started at: 2023-06-08 12:24:50.643389 +2023-06-08 12:24:50,644 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status +2023-06-08 12:24:50,647 INFO WriterThread:5514 [datastore.py:open_for_write():85] open: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/run-vrqnfbac.wandb +2023-06-08 12:24:50,648 DEBUG SenderThread:5514 [sender.py:send():369] send: header +2023-06-08 12:24:50,664 DEBUG SenderThread:5514 [sender.py:send():369] send: run +2023-06-08 12:24:50,879 INFO SenderThread:5514 [dir_watcher.py:__init__():211] watching files in: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files +2023-06-08 12:24:50,879 INFO SenderThread:5514 [sender.py:_start_run_threads():1100] run started: vrqnfbac with start time 1686227090.643913 +2023-06-08 12:24:50,879 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:24:50,880 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:24:50,882 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: check_version +2023-06-08 12:24:50,883 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: check_version +2023-06-08 12:24:50,952 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: run_start +2023-06-08 12:24:50,962 DEBUG HandlerThread:5514 [system_info.py:__init__():31] System info init +2023-06-08 12:24:50,962 DEBUG HandlerThread:5514 [system_info.py:__init__():46] System info init done +2023-06-08 12:24:50,962 INFO HandlerThread:5514 [system_monitor.py:start():181] Starting system monitor +2023-06-08 12:24:50,962 INFO SystemMonitor:5514 [system_monitor.py:_start():145] Starting system asset monitoring threads +2023-06-08 12:24:50,963 INFO HandlerThread:5514 [system_monitor.py:probe():201] Collecting system info +2023-06-08 12:24:50,963 INFO SystemMonitor:5514 [interfaces.py:start():190] Started cpu monitoring +2023-06-08 12:24:50,963 INFO SystemMonitor:5514 [interfaces.py:start():190] Started disk monitoring +2023-06-08 12:24:50,964 INFO SystemMonitor:5514 [interfaces.py:start():190] Started gpu monitoring +2023-06-08 12:24:50,965 INFO SystemMonitor:5514 [interfaces.py:start():190] Started memory monitoring +2023-06-08 12:24:50,966 INFO SystemMonitor:5514 [interfaces.py:start():190] Started network monitoring +2023-06-08 12:24:50,973 DEBUG HandlerThread:5514 [system_info.py:probe():195] Probing system +2023-06-08 12:24:50,980 DEBUG HandlerThread:5514 [system_info.py:_probe_git():180] Probing git +2023-06-08 12:24:50,992 DEBUG HandlerThread:5514 [system_info.py:_probe_git():188] Probing git done +2023-06-08 12:24:50,992 DEBUG HandlerThread:5514 [system_info.py:probe():240] Probing system done +2023-06-08 12:24:50,992 DEBUG HandlerThread:5514 [system_monitor.py:probe():210] {'os': 'Linux-5.15.0-1034-aws-x86_64-with-glibc2.29', 'python': '3.8.10', 'heartbeatAt': '2023-06-08T12:24:50.973990', 'startedAt': '2023-06-08T12:24:50.637418', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'train_all.py', 'codePath': 'train_all.py', 'git': {'remote': 'https://huggingface.co/krupalkp/custom_llm-small', 'commit': 'fa712672eb21ef9096828ca756ee5738f9743137'}, 'email': None, 'root': '/workspace/custom_llm-small', 'host': '2aaab01b09a9', 'username': 'root', 'executable': '/workspace/envs/llmenv/bin/python', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2799.9982499999996, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 3100.042, 'min': 0.0, 'max': 0.0}, {'current': 2499.998, 'min': 0.0, 'max': 0.0}, {'current': 3099.955, 'min': 0.0, 'max': 0.0}, {'current': 2499.998, 'min': 0.0, 'max': 0.0}], 'disk': {'total': 72.63036346435547, 'used': 55.228641510009766}, 'gpu': 'Tesla T4', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 15.337417602539062}} +2023-06-08 12:24:50,992 INFO HandlerThread:5514 [system_monitor.py:probe():211] Finished collecting system info +2023-06-08 12:24:50,992 INFO HandlerThread:5514 [system_monitor.py:probe():214] Publishing system info +2023-06-08 12:24:50,992 DEBUG HandlerThread:5514 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2023-06-08 12:24:50,993 DEBUG HandlerThread:5514 [system_info.py:_save_pip():67] Saving pip packages done +2023-06-08 12:24:50,993 INFO HandlerThread:5514 [system_monitor.py:probe():216] Finished publishing system info +2023-06-08 12:24:51,000 DEBUG SenderThread:5514 [sender.py:send():369] send: files +2023-06-08 12:24:51,000 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now +2023-06-08 12:24:51,005 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:24:51,005 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:24:51,075 DEBUG SenderThread:5514 [sender.py:send():369] send: telemetry +2023-06-08 12:24:51,273 INFO wandb-upload_0:5514 [upload_job.py:push():133] Uploaded file /tmp/tmp4o2jlc8twandb/n5qzpzkb-wandb-metadata.json +2023-06-08 12:24:51,882 INFO Thread-12 :5514 [dir_watcher.py:_on_file_created():272] file/dir created: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:24:51,882 INFO Thread-12 :5514 [dir_watcher.py:_on_file_created():272] file/dir created: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:24:51,882 INFO Thread-12 :5514 [dir_watcher.py:_on_file_created():272] file/dir created: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-metadata.json +2023-06-08 12:24:51,882 INFO Thread-12 :5514 [dir_watcher.py:_on_file_created():272] file/dir created: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/requirements.txt +2023-06-08 12:24:53,880 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:24:55,881 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:24:56,519 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:24:57,882 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:25:01,520 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:06,005 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:25:06,005 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:25:07,096 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:07,885 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:25:12,097 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:17,098 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:21,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:25:21,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:25:22,128 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:22,889 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/config.yaml +2023-06-08 12:25:27,216 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:32,217 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:36,015 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:25:36,016 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:25:38,058 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:43,059 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:48,060 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:50,966 DEBUG SystemMonitor:5514 [system_monitor.py:_start():159] Starting system metrics aggregation loop +2023-06-08 12:25:50,967 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:25:51,015 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:25:51,016 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:25:53,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:58,951 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:25:59,902 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:26:03,951 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:05,426 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:26:05,427 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:26:05,427 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:26:05,429 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:26:05,904 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:26:06,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:26:06,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:26:07,905 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:26:09,076 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:14,076 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:19,077 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:20,968 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:26:21,016 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:26:21,017 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:26:25,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:30,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:35,076 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:36,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:26:36,017 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:26:41,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:46,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:49,919 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:26:50,968 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:26:51,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:26:51,017 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:26:52,060 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:26:57,061 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:02,062 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:03,546 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:27:03,547 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:27:03,547 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:27:03,548 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:27:03,925 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:27:05,926 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:27:06,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:27:06,017 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:27:07,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:12,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:17,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:19,724 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:27:19,724 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:27:19,725 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:27:19,725 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:27:19,931 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:27:20,969 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:27:21,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:27:21,017 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:27:21,932 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:27:22,131 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:27,132 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:32,132 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:35,790 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:27:35,790 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:27:35,791 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:27:35,791 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:27:35,937 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:27:36,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:27:36,017 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:27:37,938 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:27:38,064 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:43,065 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:48,066 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:50,970 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:27:51,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:27:51,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:27:51,430 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:27:51,431 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:27:51,431 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:27:51,432 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:27:51,944 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:27:53,433 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:27:53,945 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:27:58,434 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:03,434 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:06,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:28:06,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:28:06,952 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:28:06,953 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:28:06,954 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:28:06,954 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:28:07,950 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:28:08,955 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:09,951 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:28:13,955 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:18,956 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:20,970 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:28:21,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:28:21,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:28:22,424 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:28:22,425 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:28:22,425 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:28:22,426 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:28:22,956 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:28:23,956 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:28:24,427 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:29,428 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:34,428 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:36,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:28:36,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:28:37,776 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:28:37,777 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:28:37,777 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:28:37,777 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:28:37,961 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:28:39,778 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:39,962 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:28:44,779 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:49,780 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:50,971 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:28:51,017 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:28:51,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:28:53,132 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:28:53,133 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:28:53,134 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:28:53,134 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:28:53,968 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:28:55,135 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:28:55,968 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:29:00,136 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:05,136 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:06,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:29:06,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:29:08,725 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:29:08,726 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:29:08,726 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:29:08,727 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:29:08,973 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:29:09,974 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:29:10,727 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:15,728 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:20,729 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:20,971 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:29:21,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:29:21,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:29:24,770 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:29:24,771 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:29:24,771 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:29:24,772 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:29:24,980 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:29:25,772 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:25,980 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:29:30,773 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:35,774 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:36,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:29:36,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:29:40,777 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:29:40,778 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:40,778 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:29:40,778 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:29:40,779 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:29:40,986 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:29:41,986 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:29:45,779 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:50,780 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:50,973 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:29:51,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:29:51,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:29:56,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:29:56,810 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:29:56,811 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:29:56,811 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:29:56,811 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:29:56,992 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:29:57,993 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:30:01,812 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:06,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:30:06,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:30:07,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:12,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:12,836 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:30:12,837 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:30:12,837 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:30:12,839 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:30:12,998 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:30:13,999 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:30:17,840 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:20,974 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:30:21,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:30:21,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:30:23,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:28,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:28,987 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:30:28,988 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:30:28,988 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:30:28,988 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:30:29,005 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:30:30,005 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:30:33,989 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:36,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:30:36,018 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:30:39,357 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:44,358 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:45,112 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:30:45,113 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:30:45,113 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:30:45,116 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:30:46,011 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:30:48,012 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:30:50,117 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:30:50,975 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:30:51,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:30:51,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:30:56,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:01,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:02,490 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:31:02,491 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:31:02,491 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:31:02,492 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:31:03,017 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:31:04,018 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:31:06,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:31:06,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:31:09,661 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:14,662 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:19,000 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:31:19,001 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:31:19,001 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:31:19,001 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:31:19,023 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:31:20,002 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:20,024 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:31:20,975 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:31:21,018 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:31:21,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:31:25,105 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:30,105 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:35,106 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:35,298 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:31:35,299 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:31:35,299 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:31:35,300 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:31:36,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:31:36,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:31:36,030 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:31:38,030 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:31:41,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:46,076 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:50,976 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:31:51,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:31:51,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:31:51,465 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:31:51,465 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:31:51,466 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:31:51,466 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:31:51,467 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:31:52,036 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:31:52,036 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:31:56,468 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:01,469 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:06,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:32:06,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:32:07,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:07,658 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:32:07,659 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:32:07,659 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:32:07,660 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:32:08,042 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:32:10,043 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:32:12,661 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:17,661 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:20,976 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:32:21,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:32:21,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:32:23,105 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:23,893 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:32:23,894 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:32:23,894 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:32:23,895 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:32:24,048 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:32:26,049 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:32:28,896 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:33,897 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:36,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:32:36,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:32:39,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:40,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:32:40,022 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:32:40,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:32:40,023 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:32:40,054 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:32:42,055 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:32:45,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:50,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:50,977 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:32:51,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:32:51,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:32:55,064 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:32:56,179 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:32:56,180 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:32:56,180 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:32:56,181 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:32:57,061 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:32:58,061 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:33:00,182 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:05,182 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:06,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:33:06,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:33:11,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:12,324 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:33:12,325 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:33:12,325 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:33:12,326 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:33:13,067 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:33:14,068 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:33:16,327 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:20,977 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:33:21,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:33:21,019 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:33:22,113 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:27,114 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:28,409 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:33:28,409 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:33:28,410 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:33:28,410 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:33:29,074 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:33:30,074 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:33:32,411 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:36,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:33:36,020 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:33:38,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:43,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:44,483 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:33:44,484 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:33:44,485 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:33:44,485 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:33:45,080 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:33:46,080 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:33:48,486 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:50,978 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:33:51,019 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:33:51,020 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:33:54,062 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:33:59,063 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:00,500 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:34:00,501 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:34:00,501 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:34:00,502 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:34:01,086 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:34:02,087 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:34:04,503 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:06,020 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:34:06,020 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:34:10,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:15,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:16,535 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:34:16,536 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:34:16,536 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:34:16,537 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:34:17,092 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:34:18,093 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:34:20,538 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:20,979 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:34:21,020 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:34:21,020 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:34:26,124 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:31,125 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:32,561 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:34:32,562 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:34:32,562 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:34:32,563 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:34:33,098 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:34:34,099 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:34:36,020 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:34:36,020 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:34:37,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:42,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:47,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:48,591 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:34:48,592 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:34:48,592 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:34:48,592 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:34:49,104 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:34:50,105 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:34:50,980 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:34:51,020 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:34:51,020 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:34:52,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:34:57,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:02,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:04,601 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:35:04,602 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:35:04,602 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:35:04,602 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:35:05,111 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:35:06,021 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:35:06,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:35:06,111 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:35:07,096 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:12,096 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:17,097 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:20,932 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:35:20,933 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:35:20,933 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:35:20,933 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:35:20,983 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:35:21,021 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:35:21,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:35:21,117 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:35:22,117 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:35:23,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:28,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:33,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:36,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:35:36,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:35:36,948 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:35:38,465 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:35:38,465 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:38,466 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:35:38,467 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:35:39,124 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:35:40,125 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:35:43,468 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:48,469 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:50,981 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:35:51,021 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:35:51,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:35:53,062 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:35:53,063 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:35:53,063 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:35:53,065 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:35:53,130 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:35:54,066 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:35:54,130 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:35:59,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:04,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:06,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:36:06,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:36:09,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:09,344 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:36:09,345 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:36:09,345 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:36:09,346 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:36:10,136 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:36:10,137 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:36:14,347 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:19,348 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:20,982 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:36:21,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:36:21,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:36:25,096 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:25,617 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:36:25,618 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:36:25,618 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:36:25,621 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:36:26,142 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:36:26,143 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:36:30,622 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:35,623 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:36,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:36:36,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:36:41,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:41,769 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:36:41,770 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:36:41,770 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:36:41,770 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:36:42,151 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:36:42,151 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:36:46,771 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:50,983 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:36:51,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:36:51,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:36:52,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:57,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:36:57,833 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:36:57,834 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:36:57,834 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:36:57,836 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:36:58,157 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:36:58,157 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:37:02,837 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:06,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:37:06,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:37:08,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:13,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:13,855 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:37:13,856 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:37:13,856 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:37:13,857 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:37:14,162 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:37:14,163 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:37:18,858 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:20,983 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:37:21,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:37:21,022 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:37:24,083 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:29,088 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:29,897 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:37:29,898 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:37:29,898 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:37:29,901 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:37:30,168 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:37:30,168 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:37:34,902 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:36,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:37:36,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:37:40,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:45,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:45,936 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:37:45,937 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:37:45,937 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:37:45,937 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:37:46,173 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:37:46,174 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:37:50,938 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:37:50,984 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:37:51,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:37:51,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:37:56,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:01,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:01,956 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:38:01,957 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:38:01,957 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:38:01,960 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:38:02,180 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:38:02,180 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:38:06,022 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:38:06,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:38:07,066 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:12,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:17,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:17,994 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:38:17,995 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:38:17,995 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:38:17,995 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:38:18,185 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:38:18,186 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:38:20,985 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:38:21,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:38:21,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:38:22,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:27,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:32,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:34,063 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:38:34,064 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:38:34,064 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:38:34,067 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:38:34,191 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:38:36,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:38:36,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:38:36,192 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:38:38,064 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:43,064 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:48,065 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:50,316 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:38:50,317 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:38:50,317 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:38:50,318 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:38:50,986 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:38:51,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:38:51,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:38:51,198 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:38:52,198 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:38:53,066 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:38:58,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:03,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:06,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:39:06,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:39:06,424 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:39:06,425 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:39:06,425 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:39:06,428 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:39:07,203 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:39:08,204 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:39:08,428 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:13,429 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:18,430 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:20,987 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:39:21,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:39:21,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:39:22,504 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:39:22,505 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:39:22,505 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:39:22,506 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:39:23,209 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:39:23,506 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:24,210 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:39:28,507 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:33,508 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:36,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:39:36,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:39:38,894 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:39:38,895 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:38,895 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:39:38,895 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:39:38,898 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:39:39,215 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:39:40,215 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:39:43,899 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:48,900 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:50,988 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:39:51,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:39:51,023 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:39:54,066 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:39:54,991 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:39:54,992 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:39:54,992 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:39:54,993 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:39:55,220 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:39:56,221 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:39:59,994 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:04,995 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:06,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:40:06,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:40:10,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:11,211 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:40:11,212 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:40:11,212 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:40:11,214 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:40:11,226 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:40:12,226 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:40:15,215 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:20,216 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:20,989 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:40:21,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:40:21,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:40:26,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:27,382 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:40:27,383 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:40:27,383 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:40:27,383 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:40:28,232 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:40:28,232 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:40:31,384 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:36,023 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:40:36,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:40:37,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:42,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:43,527 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:40:43,527 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:40:43,528 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:40:43,531 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:40:44,239 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:40:45,239 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:40:47,532 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:50,989 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:40:51,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:40:51,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:40:53,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:58,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:40:59,810 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:40:59,811 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:40:59,811 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:40:59,812 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:41:00,244 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:41:01,245 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:41:03,812 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:06,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:41:06,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:41:09,077 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:14,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:16,226 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:41:16,227 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:41:16,227 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:41:16,236 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:41:16,252 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:41:17,252 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:41:19,238 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:20,990 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:41:21,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:41:21,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:41:25,114 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:30,115 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:33,923 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:41:33,924 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:41:33,925 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:41:33,925 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:41:34,258 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:41:35,258 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:41:35,926 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:36,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:41:36,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:41:41,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:46,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:49,973 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:41:49,974 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:41:49,974 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:41:49,976 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:41:50,264 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:41:50,991 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:41:51,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:41:51,024 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:41:51,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:41:51,264 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:41:56,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:01,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:06,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:42:06,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:42:06,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:42:06,078 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:42:06,078 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:42:06,079 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:42:06,269 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:42:07,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:07,270 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:42:12,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:17,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:20,991 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:42:21,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:42:21,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:42:22,099 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:42:22,099 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:22,100 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:42:22,100 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:42:22,100 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:42:22,275 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:42:23,275 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:42:27,103 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:32,103 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:36,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:42:36,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:42:38,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:38,137 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:42:38,138 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:42:38,138 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:42:38,139 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:42:38,281 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:42:39,281 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:42:43,140 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:48,140 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:50,992 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:42:51,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:42:51,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:42:54,066 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:42:54,228 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:42:54,229 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:42:54,229 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:42:54,230 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:42:54,287 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:42:55,288 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:42:59,237 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:04,238 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:06,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:43:06,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:43:10,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:10,224 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:43:10,225 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:43:10,225 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:43:10,225 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:43:10,295 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:43:11,295 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:43:15,226 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:20,227 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:20,993 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:43:21,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:43:21,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:43:26,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:26,254 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:43:26,255 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:43:26,256 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:43:26,256 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:43:26,302 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:43:27,302 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:43:31,257 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:36,024 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:43:36,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:43:37,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:42,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:42,305 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:43:42,306 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:43:42,306 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:43:42,306 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:43:42,308 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:43:43,308 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:43:47,307 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:50,993 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:43:51,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:43:51,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:43:53,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:58,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:43:58,697 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:43:58,698 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:43:58,698 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:43:58,699 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:43:59,315 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:44:01,316 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:44:03,700 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:06,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:44:06,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:44:09,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:14,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:14,802 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:44:14,803 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:44:14,804 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:44:14,804 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:44:15,322 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:44:17,323 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:44:19,805 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:20,994 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:44:21,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:44:21,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:44:25,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:30,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:30,871 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:44:30,872 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:44:30,872 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:44:30,872 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:44:31,328 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:44:33,329 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:44:35,873 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:36,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:44:36,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:44:41,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:46,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:46,911 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:44:46,912 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:44:46,912 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:44:46,912 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:44:47,334 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:44:49,335 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:44:50,994 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:44:51,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:44:51,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:44:51,097 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:44:56,098 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:01,098 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:03,180 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:45:03,181 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:45:03,181 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:45:03,182 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:45:03,341 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:45:05,342 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:45:06,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:45:06,025 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:45:07,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:12,069 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:17,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:19,384 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:45:19,385 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:45:19,385 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:45:19,386 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:45:20,348 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:45:20,995 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:45:21,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:45:21,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:45:21,348 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:45:22,118 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:27,119 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:32,120 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:35,455 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:45:35,456 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:45:35,456 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:45:35,456 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:45:36,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:45:36,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:45:36,354 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:45:37,354 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:45:38,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:43,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:48,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:50,996 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:45:51,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:45:51,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:45:51,797 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:45:51,798 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:45:51,798 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:45:51,799 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:45:52,360 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:45:53,360 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:45:53,799 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:45:58,801 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:03,802 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:06,025 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:46:06,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:46:08,013 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:46:09,050 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:46:09,050 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:09,051 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:46:09,052 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:46:09,366 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:46:11,367 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:46:14,053 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:19,053 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:20,997 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:46:21,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:46:21,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:46:24,077 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:24,154 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:46:24,155 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:46:24,155 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:46:24,156 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:46:24,372 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:46:25,373 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:46:29,157 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:34,158 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:36,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:46:36,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:46:40,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:40,263 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:46:40,264 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:46:40,264 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:46:40,265 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:46:40,381 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:46:41,381 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:46:45,266 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:50,267 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:50,997 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:46:51,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:46:51,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:46:56,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:46:56,328 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:46:56,329 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:46:56,329 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:46:56,330 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:46:56,387 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:46:57,387 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:47:01,331 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:06,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:47:06,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:47:07,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:12,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:12,395 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:47:12,397 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:47:12,397 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:47:12,397 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:47:13,397 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:47:13,397 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:47:17,399 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:20,998 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:47:21,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:47:21,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:47:23,067 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:28,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:28,464 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:47:28,465 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:47:28,465 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:47:28,466 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:47:29,403 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:47:29,403 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:47:33,467 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:36,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:47:36,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:47:39,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:44,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:44,513 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:47:44,514 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:47:44,514 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:47:44,515 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:47:45,408 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:47:45,409 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:47:49,516 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:47:51,001 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:47:51,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:47:51,026 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:47:55,077 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:00,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:00,542 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:48:00,543 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:48:00,543 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:48:00,544 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:48:01,414 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:48:01,415 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:48:05,544 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:06,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:48:06,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:48:11,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:16,083 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:16,881 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:48:16,882 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:48:16,882 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:48:16,882 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:48:17,420 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:48:19,421 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:48:20,999 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:48:21,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:48:21,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:48:22,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:27,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:32,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:32,922 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:48:32,923 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:48:32,923 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:48:32,923 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:48:33,426 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:48:35,427 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:48:36,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:48:36,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:48:38,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:43,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:48,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:49,119 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:48:49,120 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:48:49,120 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:48:49,121 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:48:49,432 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:48:51,000 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:48:51,026 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:48:51,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:48:51,433 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:48:54,077 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:48:59,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:04,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:05,328 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:49:05,328 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:49:05,329 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:49:05,331 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:49:05,438 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:49:06,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:49:06,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:49:07,439 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:49:09,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:14,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:19,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:21,001 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:49:21,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:49:21,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:49:21,468 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:49:21,469 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:49:21,469 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:49:21,470 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:49:22,447 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:49:23,447 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:49:24,471 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:29,471 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:34,472 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:36,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:49:36,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:49:37,553 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:49:37,554 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:49:37,554 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:49:37,556 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:49:38,452 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:49:39,453 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:49:39,557 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:44,558 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:49,559 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:51,001 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:49:51,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:49:51,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:49:53,605 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:49:53,606 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:49:53,606 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:49:53,607 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:49:54,460 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:49:54,607 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:49:55,461 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:49:59,608 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:04,609 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:06,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:50:06,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:50:09,867 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:50:09,868 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:09,870 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:50:09,870 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:50:09,873 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:50:10,467 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:50:11,467 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:50:14,874 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:19,875 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:21,002 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:50:21,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:50:21,027 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:50:25,109 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:26,102 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:50:26,103 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:50:26,103 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:50:26,104 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:50:26,472 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:50:27,472 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:50:31,105 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:36,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:50:36,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:50:37,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:42,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:42,245 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:50:42,246 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:50:42,247 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:50:42,247 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:50:42,478 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:50:43,478 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:50:47,248 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:51,003 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:50:51,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:50:51,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:50:53,079 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:58,080 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:50:58,504 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:50:58,505 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:50:58,506 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:50:58,506 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:50:59,486 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:50:59,486 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:51:03,507 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:06,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:51:06,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:51:09,070 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:14,071 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:14,636 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:51:14,637 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:51:14,637 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:51:14,639 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:51:15,494 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:51:15,494 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:51:19,640 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:21,003 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:51:21,027 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:51:21,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:51:25,083 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:30,083 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:30,715 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:51:30,716 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:51:30,716 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:51:30,716 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:51:31,500 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:51:31,501 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:51:35,717 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:36,028 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:51:36,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:51:41,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:46,082 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:46,786 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:51:46,787 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:51:46,787 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:51:46,789 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:51:47,506 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:51:47,507 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:51:51,004 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:51:51,028 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:51:51,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:51:52,072 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:51:57,073 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:02,074 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:02,965 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:52:02,966 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:52:02,966 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:52:02,966 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:52:03,512 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:52:03,512 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:52:06,028 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:52:06,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:52:07,075 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:12,076 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:17,077 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:19,048 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:52:19,049 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:52:19,049 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:52:19,051 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:52:19,520 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:52:21,004 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:52:21,028 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:52:21,028 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:52:21,521 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:52:22,108 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:27,109 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:31,525 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:52:33,051 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:33,525 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:52:36,030 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:52:36,030 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:52:38,081 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:39,527 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:52:43,269 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:48,270 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:51,005 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:52:51,087 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:52:51,088 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:52:54,134 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:52:59,135 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:04,136 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:06,032 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:53:06,033 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:53:10,089 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:15,090 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:20,090 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:21,006 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:53:21,033 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:53:21,033 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:53:23,544 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:53:25,973 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:26,341 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:53:26,343 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:53:26,343 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:53:26,343 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:53:26,545 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:53:27,546 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:53:31,398 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:36,068 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:53:36,068 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:53:37,130 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:42,131 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:47,132 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:51,006 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:53:51,038 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:53:51,039 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:53:53,085 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:53:58,086 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:03,087 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:06,038 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:54:06,039 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:54:08,571 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:09,560 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:54:13,561 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:54:13,608 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:15,562 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:54:19,506 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:19,563 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:54:21,007 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:54:21,078 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:54:21,079 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:54:25,128 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:30,131 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:33,568 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:54:35,860 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:36,043 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:54:36,044 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:54:41,087 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:46,088 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:51,008 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:54:51,043 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:54:51,044 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:54:51,099 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:54:56,099 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:01,100 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:06,044 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:55:06,044 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:55:07,092 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:12,093 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:17,094 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:21,008 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:55:21,043 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:55:21,044 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:55:23,091 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:28,092 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:33,093 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:36,044 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:55:36,045 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:55:39,087 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:44,088 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:49,089 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:51,009 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:55:51,044 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:55:51,045 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:55:54,100 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:55:59,100 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:04,101 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:06,044 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:56:06,045 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:56:10,096 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:15,097 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:20,098 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:21,010 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:56:21,044 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:56:21,045 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:56:26,098 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:31,099 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:32,234 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: partial_history +2023-06-08 12:56:32,235 DEBUG SenderThread:5514 [sender.py:send():369] send: history +2023-06-08 12:56:32,235 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: summary_record +2023-06-08 12:56:32,235 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:56:32,617 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:56:33,618 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:56:35,618 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:56:36,045 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: stop_status +2023-06-08 12:56:36,045 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: stop_status +2023-06-08 12:56:36,459 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:36,502 DEBUG SenderThread:5514 [sender.py:send():369] send: exit +2023-06-08 12:56:36,502 INFO SenderThread:5514 [sender.py:send_exit():574] handling exit code: 1 +2023-06-08 12:56:36,502 INFO SenderThread:5514 [sender.py:send_exit():576] handling runtime: 1905 +2023-06-08 12:56:36,502 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:56:36,502 INFO SenderThread:5514 [sender.py:send_exit():582] send defer +2023-06-08 12:56:36,502 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,502 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 0 +2023-06-08 12:56:36,503 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,503 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 0 +2023-06-08 12:56:36,503 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 1 +2023-06-08 12:56:36,503 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,503 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 1 +2023-06-08 12:56:36,503 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,503 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 1 +2023-06-08 12:56:36,503 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 2 +2023-06-08 12:56:36,503 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,503 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 2 +2023-06-08 12:56:36,503 INFO HandlerThread:5514 [system_monitor.py:finish():190] Stopping system monitor +2023-06-08 12:56:36,503 DEBUG SystemMonitor:5514 [system_monitor.py:_start():166] Finished system metrics aggregation loop +2023-06-08 12:56:36,504 INFO HandlerThread:5514 [interfaces.py:finish():202] Joined cpu monitor +2023-06-08 12:56:36,504 DEBUG SystemMonitor:5514 [system_monitor.py:_start():170] Publishing last batch of metrics +2023-06-08 12:56:36,504 INFO HandlerThread:5514 [interfaces.py:finish():202] Joined disk monitor +2023-06-08 12:56:36,507 INFO HandlerThread:5514 [interfaces.py:finish():202] Joined gpu monitor +2023-06-08 12:56:36,507 INFO HandlerThread:5514 [interfaces.py:finish():202] Joined memory monitor +2023-06-08 12:56:36,507 INFO HandlerThread:5514 [interfaces.py:finish():202] Joined network monitor +2023-06-08 12:56:36,507 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,508 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 2 +2023-06-08 12:56:36,508 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 3 +2023-06-08 12:56:36,508 DEBUG SenderThread:5514 [sender.py:send():369] send: stats +2023-06-08 12:56:36,508 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,508 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 3 +2023-06-08 12:56:36,508 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,508 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 3 +2023-06-08 12:56:36,508 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 4 +2023-06-08 12:56:36,508 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,508 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 4 +2023-06-08 12:56:36,509 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,509 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 4 +2023-06-08 12:56:36,509 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 5 +2023-06-08 12:56:36,509 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,509 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 5 +2023-06-08 12:56:36,509 DEBUG SenderThread:5514 [sender.py:send():369] send: summary +2023-06-08 12:56:36,509 INFO SenderThread:5514 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-08 12:56:36,509 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,510 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 5 +2023-06-08 12:56:36,510 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 6 +2023-06-08 12:56:36,510 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,510 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 6 +2023-06-08 12:56:36,510 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,510 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 6 +2023-06-08 12:56:36,510 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 7 +2023-06-08 12:56:36,510 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: status_report +2023-06-08 12:56:36,510 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:36,510 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 7 +2023-06-08 12:56:36,510 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:36,510 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 7 +2023-06-08 12:56:36,619 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:56:37,416 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 8 +2023-06-08 12:56:37,417 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:37,417 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 8 +2023-06-08 12:56:37,417 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:37,417 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 8 +2023-06-08 12:56:37,417 INFO SenderThread:5514 [job_builder.py:build():232] Attempting to build job artifact +2023-06-08 12:56:37,417 INFO SenderThread:5514 [job_builder.py:build():256] is repo sourced job +2023-06-08 12:56:37,420 INFO SenderThread:5514 [job_builder.py:build():297] adding wandb-job metadata file +2023-06-08 12:56:37,424 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 9 +2023-06-08 12:56:37,425 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:37,425 DEBUG SenderThread:5514 [sender.py:send():369] send: artifact +2023-06-08 12:56:37,425 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 9 +2023-06-08 12:56:37,502 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: poll_exit +2023-06-08 12:56:37,619 INFO Thread-12 :5514 [dir_watcher.py:_on_file_modified():289] file/dir modified: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:56:38,024 INFO wandb-upload_0:5514 [upload_job.py:push():91] Uploaded file /root/.local/share/wandb/artifacts/staging/tmp6y7gwc0s +2023-06-08 12:56:38,126 INFO wandb-upload_1:5514 [upload_job.py:push():91] Uploaded file /root/.local/share/wandb/artifacts/staging/tmp_ezind0n +2023-06-08 12:56:38,870 INFO SenderThread:5514 [sender.py:send_artifact():1450] sent artifact job-https___huggingface.co_krupalkp_custom_llm-small_train_all.py - {'id': 'QXJ0aWZhY3Q6NDgxNTIwNTQx', 'digest': 'c013af7f7dccf67590d04053871e1b07', 'state': 'PENDING', 'aliases': [], 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjc1NDcwODc5', 'latestArtifact': None}, 'version': 'latest'} +2023-06-08 12:56:38,870 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:38,870 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 9 +2023-06-08 12:56:38,870 INFO SenderThread:5514 [dir_watcher.py:finish():359] shutting down directory watcher +2023-06-08 12:56:39,619 INFO SenderThread:5514 [dir_watcher.py:finish():389] scan: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files +2023-06-08 12:56:39,620 INFO SenderThread:5514 [dir_watcher.py:finish():403] scan save: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json wandb-summary.json +2023-06-08 12:56:39,620 INFO SenderThread:5514 [dir_watcher.py:finish():403] scan save: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-metadata.json wandb-metadata.json +2023-06-08 12:56:39,620 INFO SenderThread:5514 [dir_watcher.py:finish():403] scan save: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log output.log +2023-06-08 12:56:39,622 INFO SenderThread:5514 [dir_watcher.py:finish():403] scan save: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/requirements.txt requirements.txt +2023-06-08 12:56:39,625 INFO SenderThread:5514 [dir_watcher.py:finish():403] scan save: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/config.yaml config.yaml +2023-06-08 12:56:39,626 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 10 +2023-06-08 12:56:39,628 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: poll_exit +2023-06-08 12:56:39,628 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:39,630 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 10 +2023-06-08 12:56:39,631 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:39,631 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 10 +2023-06-08 12:56:39,631 INFO SenderThread:5514 [file_pusher.py:finish():159] shutting down file pusher +2023-06-08 12:56:39,877 INFO wandb-upload_0:5514 [upload_job.py:push():133] Uploaded file /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/wandb-summary.json +2023-06-08 12:56:39,919 INFO wandb-upload_3:5514 [upload_job.py:push():133] Uploaded file /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/config.yaml +2023-06-08 12:56:39,926 INFO wandb-upload_2:5514 [upload_job.py:push():133] Uploaded file /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/requirements.txt +2023-06-08 12:56:39,936 INFO wandb-upload_1:5514 [upload_job.py:push():133] Uploaded file /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/files/output.log +2023-06-08 12:56:40,137 INFO Thread-11 :5514 [sender.py:transition_state():602] send defer: 11 +2023-06-08 12:56:40,137 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:40,137 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 11 +2023-06-08 12:56:40,138 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:40,138 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 11 +2023-06-08 12:56:40,138 INFO SenderThread:5514 [file_pusher.py:join():164] waiting for file pusher +2023-06-08 12:56:40,138 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 12 +2023-06-08 12:56:40,138 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:40,138 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 12 +2023-06-08 12:56:40,138 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:40,138 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 12 +2023-06-08 12:56:40,375 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 13 +2023-06-08 12:56:40,375 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:40,375 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 13 +2023-06-08 12:56:40,375 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:40,375 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 13 +2023-06-08 12:56:40,376 INFO SenderThread:5514 [sender.py:transition_state():602] send defer: 14 +2023-06-08 12:56:40,376 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: defer +2023-06-08 12:56:40,376 INFO HandlerThread:5514 [handler.py:handle_request_defer():170] handle defer: 14 +2023-06-08 12:56:40,376 DEBUG SenderThread:5514 [sender.py:send():369] send: final +2023-06-08 12:56:40,376 DEBUG SenderThread:5514 [sender.py:send():369] send: footer +2023-06-08 12:56:40,376 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: defer +2023-06-08 12:56:40,376 INFO SenderThread:5514 [sender.py:send_request_defer():598] handle sender defer: 14 +2023-06-08 12:56:40,377 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: poll_exit +2023-06-08 12:56:40,377 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: poll_exit +2023-06-08 12:56:40,377 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: server_info +2023-06-08 12:56:40,378 DEBUG SenderThread:5514 [sender.py:send_request():396] send_request: server_info +2023-06-08 12:56:40,380 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: get_summary +2023-06-08 12:56:40,380 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: sampled_history +2023-06-08 12:56:40,418 INFO MainThread:5514 [wandb_run.py:_footer_history_summary_info():3467] rendering history +2023-06-08 12:56:40,419 INFO MainThread:5514 [wandb_run.py:_footer_history_summary_info():3499] rendering summary +2023-06-08 12:56:40,419 INFO MainThread:5514 [wandb_run.py:_footer_sync_info():3426] logging synced files +2023-06-08 12:56:40,419 DEBUG HandlerThread:5514 [handler.py:handle_request():144] handle_request: shutdown +2023-06-08 12:56:40,420 INFO HandlerThread:5514 [handler.py:finish():854] shutting down handler +2023-06-08 12:56:41,378 INFO WriterThread:5514 [datastore.py:close():298] close: /workspace/custom_llm-small/wandb/run-20230608_122450-vrqnfbac/run-vrqnfbac.wandb +2023-06-08 12:56:41,418 INFO SenderThread:5514 [sender.py:finish():1526] shutting down sender +2023-06-08 12:56:41,418 INFO SenderThread:5514 [file_pusher.py:finish():159] shutting down file pusher +2023-06-08 12:56:41,418 INFO SenderThread:5514 [file_pusher.py:join():164] waiting for file pusher