diff --git "a/wandb/run-20240921_115259-p9bvnzls/logs/debug-internal.log" "b/wandb/run-20240921_115259-p9bvnzls/logs/debug-internal.log"
new file mode 100644--- /dev/null
+++ "b/wandb/run-20240921_115259-p9bvnzls/logs/debug-internal.log"
@@ -0,0 +1,2859 @@
+2024-09-21 11:52:59,201 INFO    StreamThr :1363837 [internal.py:wandb_internal():85] W&B internal server running at pid: 1363837, started at: 2024-09-21 11:52:59.199657
+2024-09-21 11:52:59,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status
+2024-09-21 11:52:59,204 INFO    WriterThread:1363837 [datastore.py:open_for_write():87] open: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/run-p9bvnzls.wandb
+2024-09-21 11:52:59,206 DEBUG   SenderThread:1363837 [sender.py:send():379] send: header
+2024-09-21 11:52:59,227 DEBUG   SenderThread:1363837 [sender.py:send():379] send: run
+2024-09-21 11:52:59,736 INFO    SenderThread:1363837 [dir_watcher.py:__init__():211] watching files in: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files
+2024-09-21 11:52:59,737 INFO    SenderThread:1363837 [sender.py:_start_run_threads():1188] run started: p9bvnzls with start time 1726890779.202056
+2024-09-21 11:52:59,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: check_version
+2024-09-21 11:52:59,744 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: check_version
+2024-09-21 11:52:59,966 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: run_start
+2024-09-21 11:53:00,054 DEBUG   HandlerThread:1363837 [system_info.py:__init__():26] System info init
+2024-09-21 11:53:00,054 DEBUG   HandlerThread:1363837 [system_info.py:__init__():41] System info init done
+2024-09-21 11:53:00,054 INFO    HandlerThread:1363837 [system_monitor.py:start():194] Starting system monitor
+2024-09-21 11:53:00,054 INFO    SystemMonitor:1363837 [system_monitor.py:_start():158] Starting system asset monitoring threads
+2024-09-21 11:53:00,054 INFO    HandlerThread:1363837 [system_monitor.py:probe():214] Collecting system info
+2024-09-21 11:53:00,055 INFO    SystemMonitor:1363837 [interfaces.py:start():188] Started cpu monitoring
+2024-09-21 11:53:00,055 INFO    SystemMonitor:1363837 [interfaces.py:start():188] Started disk monitoring
+2024-09-21 11:53:00,056 INFO    SystemMonitor:1363837 [interfaces.py:start():188] Started gpu monitoring
+2024-09-21 11:53:00,056 INFO    SystemMonitor:1363837 [interfaces.py:start():188] Started memory monitoring
+2024-09-21 11:53:00,057 INFO    SystemMonitor:1363837 [interfaces.py:start():188] Started network monitoring
+2024-09-21 11:53:00,216 DEBUG   HandlerThread:1363837 [system_info.py:probe():152] Probing system
+2024-09-21 11:53:00,219 DEBUG   HandlerThread:1363837 [system_info.py:_probe_git():137] Probing git
+2024-09-21 11:53:00,237 DEBUG   HandlerThread:1363837 [system_info.py:_probe_git():145] Probing git done
+2024-09-21 11:53:00,237 DEBUG   HandlerThread:1363837 [system_info.py:probe():200] Probing system done
+2024-09-21 11:53:00,237 DEBUG   HandlerThread:1363837 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-09-21T03:53:00.216617', 'startedAt': '2024-09-21T03:52:59.188378', 'docker': None, 'cuda': None, 'args': ('--local_rank=0', '--model_name_or_path', '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', '--train_datasets', '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', '--output_dir', '../outputs/dpo_tiv2t_1.5k_base', '--train_data_files', 'extracted_preference_1.5k_washed.json', '--train_template', 'NExTQA_preference', '--train_split', 'train', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '1', '--gradient_accumulation_steps', '1'), 'state': 'running', 'program': '-m align_anything.trainers.tiv_to_t.dpo', 'codePathLocal': None, 'git': {'remote': 'git@github.com-hantao:htlou/align-anything.git', 'commit': '9eff99b0ee34f7265b8a2e1824650634c898f654'}, 'email': '2200017789@stu.pku.edu.cn', 'root': '/aifs4su/yaodong/projects/hantao/dev_cham/align-anything', 'host': 'dgx-122', 'username': 'yangyaodong', 'executable': '/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 2053.164705357143, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2999.728, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2999.218, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3000.001, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2900.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2900.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2993.46, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2899.998, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 1758.8549690246582, 'used': 114.61069869995117}}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2015.564224243164}}
+2024-09-21 11:53:00,237 INFO    HandlerThread:1363837 [system_monitor.py:probe():224] Finished collecting system info
+2024-09-21 11:53:00,237 INFO    HandlerThread:1363837 [system_monitor.py:probe():227] Publishing system info
+2024-09-21 11:53:00,237 DEBUG   HandlerThread:1363837 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment
+2024-09-21 11:53:00,738 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/conda-environment.yaml
+2024-09-21 11:53:06,061 DEBUG   HandlerThread:1363837 [system_info.py:_save_conda():224] Saving conda packages done
+2024-09-21 11:53:06,064 INFO    HandlerThread:1363837 [system_monitor.py:probe():229] Finished publishing system info
+2024-09-21 11:53:06,082 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:06,082 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: keepalive
+2024-09-21 11:53:06,083 DEBUG   SenderThread:1363837 [sender.py:send():379] send: files
+2024-09-21 11:53:06,083 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now
+2024-09-21 11:53:06,572 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: python_packages
+2024-09-21 11:53:06,573 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: python_packages
+2024-09-21 11:53:06,573 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:53:06,575 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:53:06,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:06,742 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/conda-environment.yaml
+2024-09-21 11:53:06,742 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-metadata.json
+2024-09-21 11:53:06,742 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/requirements.txt
+2024-09-21 11:53:06,862 DEBUG   SenderThread:1363837 [sender.py:send():379] send: telemetry
+2024-09-21 11:53:07,098 INFO    wandb-upload_0:1363837 [upload_job.py:push():130] Uploaded file /tmp/tmpad6t8jvgwandb/yvnc7p24-wandb-metadata.json
+2024-09-21 11:53:07,574 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:07,743 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:08,574 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:09,574 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:09,744 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:10,574 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:10,866 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:11,574 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:12,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:13,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:14,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:15,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:16,389 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:16,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:17,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:17,749 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:18,575 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:19,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:20,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:21,573 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:53:21,574 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:53:21,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:21,915 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:22,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:23,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:24,297 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:53:24,297 DEBUG   SenderThread:1363837 [sender.py:send():379] send: telemetry
+2024-09-21 11:53:24,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:25,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:25,754 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:26,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:27,574 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:27,576 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:32,578 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:33,759 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/config.yaml
+2024-09-21 11:53:34,853 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:35,852 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:36,573 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:53:36,574 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:53:36,853 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:37,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:37,852 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:38,409 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:53:38,410 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:53:38,410 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:53:38,412 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:53:38,764 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:53:38,852 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:39,764 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:41,766 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:43,413 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:48,414 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:50,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:51,329 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:51,573 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:53:51,574 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:53:52,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:52,585 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:53:52,585 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:53:52,586 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:53:52,588 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:53:52,774 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:53:53,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:53,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:54,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:55,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:55,776 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:53:56,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:57,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:58,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:53:59,329 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:53:59,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:00,057 DEBUG   SystemMonitor:1363837 [system_monitor.py:_start():172] Starting system metrics aggregation loop
+2024-09-21 11:54:00,060 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:54:00,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:01,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:02,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:03,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:04,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:04,633 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:04,672 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:54:04,673 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:54:04,674 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:54:04,675 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:54:04,783 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:54:05,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:06,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:06,573 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:54:06,574 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:54:07,785 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:54:09,840 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:13,006 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:14,006 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:14,733 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:54:14,733 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:54:14,734 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:54:14,738 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:54:14,790 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:54:15,006 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:15,738 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:16,006 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:17,006 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:17,792 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:54:20,747 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:25,619 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:54:25,619 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:54:25,661 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:25,866 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:26,618 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:27,753 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:27,753 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:54:27,753 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:54:27,754 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:54:27,755 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:54:27,799 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:54:28,705 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:29,705 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:29,800 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:54:30,061 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:54:30,705 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:31,619 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:36,619 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:36,744 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:37,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:38,736 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:54:38,737 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:54:38,737 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:54:38,740 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:54:38,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:38,807 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:54:39,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:40,618 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:54:40,618 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:54:40,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:41,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:41,744 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:41,809 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:54:46,744 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:50,361 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:51,361 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:52,361 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:52,416 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:54:52,417 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:52,417 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:54:52,417 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:54:52,418 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:54:52,817 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:54:53,373 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:53,818 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:54:55,819 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:54:57,419 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:54:59,218 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:54:59,219 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:54:59,219 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:55:00,065 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:55:00,217 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:01,300 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:01,301 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:55:01,301 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:55:01,302 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:55:01,304 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:55:01,824 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:55:02,257 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:03,257 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:03,304 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:03,825 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:55:04,257 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:05,258 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:09,219 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:10,931 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:11,930 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:13,071 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:13,073 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:55:13,073 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:55:13,073 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:55:13,074 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:55:13,833 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:55:14,071 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:14,217 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:55:14,218 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:55:14,530 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:15,071 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:15,834 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:55:19,531 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:21,266 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:22,266 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:23,237 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:55:23,238 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:55:23,238 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:55:23,240 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:55:23,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:23,841 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:55:24,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:25,241 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:25,266 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:25,842 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:55:26,274 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:27,274 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:30,065 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:55:30,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:30,743 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:55:30,743 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:55:31,041 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:31,742 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:32,793 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:32,793 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:55:32,793 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:55:32,794 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:55:32,795 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:55:32,848 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:55:33,751 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:34,751 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:35,850 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:55:36,795 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:41,796 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:46,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:46,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:55:46,286 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:55:47,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:47,524 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:48,228 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:55:48,229 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:55:48,229 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:55:48,231 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:55:48,284 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:48,860 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:55:49,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:49,860 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:55:50,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:51,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:52,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:53,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:53,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:54,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:55,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:56,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:58,871 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:55:59,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:55:59,871 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:00,067 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:56:01,097 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:01,097 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:56:01,098 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:56:01,098 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:56:01,099 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:56:01,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:56:01,286 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:56:01,870 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:56:02,052 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:03,052 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:03,871 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:56:04,052 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:04,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:05,052 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:06,052 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:07,052 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:08,053 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:09,053 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:10,053 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:10,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:11,209 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:56:11,209 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:11,209 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:56:11,210 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:56:11,212 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:56:11,877 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:56:12,168 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:13,168 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:13,879 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:56:14,168 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:15,168 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:16,168 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:16,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:21,271 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:21,272 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:56:21,272 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:56:21,529 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:22,270 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:23,481 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:56:23,481 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:23,481 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:56:23,482 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:56:23,483 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:56:23,887 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:56:24,431 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:25,431 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:25,889 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:56:26,431 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:27,271 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:27,431 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:28,431 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:29,431 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:30,071 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:56:33,073 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:33,620 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:34,620 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:35,729 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:56:35,729 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:35,729 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:56:35,730 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:56:35,732 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:56:35,898 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:56:36,270 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:56:36,271 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:56:36,687 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:37,687 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:37,900 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:56:38,505 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:43,506 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:44,791 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:45,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:46,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:47,021 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:56:47,022 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:56:47,022 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:56:47,023 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:56:47,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:47,909 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:56:48,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:49,024 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:49,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:49,911 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:56:53,414 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:56:53,414 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:56:53,460 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:54,414 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:54,708 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:56:55,608 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:56:55,609 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:55,609 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:56:55,610 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:56:55,611 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:56:55,916 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:56:56,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:56,918 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:56:57,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:58,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:59,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:56:59,791 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:00,070 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:57:00,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:01,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:02,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:03,564 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:05,072 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:05,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:06,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:07,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:08,192 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:57:08,193 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:57:08,193 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:57:08,196 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:57:08,414 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:57:08,414 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:57:08,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:08,928 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:57:08,928 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:57:09,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:10,658 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:10,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:11,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:12,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:13,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:14,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:15,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:15,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:16,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:17,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:18,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:19,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:20,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:21,702 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:21,745 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:57:21,745 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:57:21,746 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:57:21,747 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:57:21,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:21,937 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:57:22,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:22,937 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:57:23,414 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:57:23,414 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:57:23,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:24,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:27,696 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:29,950 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:30,072 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:57:30,949 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:32,068 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:32,069 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:57:32,069 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:57:32,070 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:57:32,071 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:57:32,943 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:57:32,944 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:57:33,025 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:33,071 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:38,072 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:41,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:41,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:57:41,143 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:57:42,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:43,102 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:43,148 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:57:43,149 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:43,149 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:57:43,149 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:57:43,151 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:57:43,951 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:57:44,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:44,952 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:57:45,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:46,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:47,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:48,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:48,143 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:52,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:53,712 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:53,714 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:57:54,741 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:57:54,741 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:54,741 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:57:54,742 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:57:54,744 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:57:54,960 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:57:55,712 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:56,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:57:56,142 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:57:56,712 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:56,962 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:57:57,712 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:58,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:59,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:57:59,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:00,075 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:58:00,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:01,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:04,269 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:05,267 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:05,268 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:06,380 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:06,381 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:58:06,381 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:58:06,382 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:58:06,383 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:58:06,971 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:58:06,972 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:58:07,336 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:08,336 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:09,336 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:10,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:11,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:58:11,142 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:58:11,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:11,414 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:12,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:13,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:14,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:15,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:16,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:17,057 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:17,096 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:58:17,097 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:58:17,098 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:58:17,099 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:58:17,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:17,979 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:58:18,337 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:18,979 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:58:22,100 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:25,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:26,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:58:26,142 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:58:26,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:27,402 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:27,925 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:58:27,925 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:58:27,926 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:58:27,928 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:58:27,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:27,987 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:58:28,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:28,988 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:58:29,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:30,075 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:58:30,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:32,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:37,938 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:39,798 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:40,798 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:41,142 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:58:41,142 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:58:41,870 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:41,871 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:58:41,872 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:58:41,872 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:58:41,873 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:58:41,998 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:58:42,870 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:42,999 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:58:43,870 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:43,874 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:48,874 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:50,962 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:51,961 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:52,789 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:58:52,789 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:58:52,790 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:58:52,791 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:58:52,961 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:53,008 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:58:53,961 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:58:54,791 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:58:55,009 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:58:59,792 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:00,077 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:59:00,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:00,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:59:00,589 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:59:01,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:02,655 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:03,158 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:59:03,159 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:59:03,160 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:59:03,161 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:59:03,655 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:04,018 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:59:04,655 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:05,019 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:59:05,162 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:05,655 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:06,655 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:07,656 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:08,656 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:09,656 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:10,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:10,656 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:13,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:14,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:15,193 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:59:15,193 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:15,193 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:59:15,194 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:59:15,196 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:59:15,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:15,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:59:15,588 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:59:16,029 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:59:16,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:17,030 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:59:17,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:18,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:19,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:20,838 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:25,838 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:26,949 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:27,949 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:29,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:59:29,077 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:29,077 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:59:29,078 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:59:29,079 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:59:30,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:30,042 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:59:30,079 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 11:59:30,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:59:30,588 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:59:30,871 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:31,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:31,043 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:59:32,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:33,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:34,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:35,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:35,949 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:36,036 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:38,005 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:39,003 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:40,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:59:40,046 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:40,046 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:59:40,046 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:59:40,047 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:59:40,052 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:59:41,003 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:41,048 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:41,053 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:59:42,003 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:46,049 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:48,705 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 11:59:48,705 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:48,705 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 11:59:49,704 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:50,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 11:59:50,617 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 11:59:50,618 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 11:59:50,619 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 11:59:50,704 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:51,062 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 11:59:51,619 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 11:59:51,704 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 11:59:53,064 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 11:59:56,620 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:00,080 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:00:02,081 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:03,519 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:03,704 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:00:03,704 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:00:04,518 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:05,488 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:00:05,489 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:00:05,489 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:00:05,490 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:00:05,518 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:06,075 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:00:06,518 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:07,076 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:00:07,491 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:07,518 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:12,492 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:16,703 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:17,701 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:17,703 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:18,753 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:00:18,753 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:00:18,753 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:18,753 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:00:18,754 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:00:18,755 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:00:18,755 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:00:19,087 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:00:19,711 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:20,711 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:21,089 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:00:23,079 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:28,080 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:30,081 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:00:30,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:31,285 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:32,377 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:32,377 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:00:32,377 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:00:32,378 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:00:32,379 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:00:33,100 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:00:33,100 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:00:33,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:33,379 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:33,711 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:00:33,711 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:00:34,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:35,331 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:36,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:37,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:39,286 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:39,392 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:40,392 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:41,464 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:00:41,465 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:41,465 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:00:41,466 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:00:41,467 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:00:42,108 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:00:42,420 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:43,109 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:00:43,421 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:44,421 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:44,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:45,421 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:50,287 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:51,439 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:51,439 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:00:51,440 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:00:52,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:53,565 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:00:53,566 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:53,567 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:00:53,567 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:00:53,569 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:00:54,121 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:00:54,517 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:55,122 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:00:55,517 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:55,570 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:00:56,517 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:57,517 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:58,517 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:00:59,517 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:00,083 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:01:01,085 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:04,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:05,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:06,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:01:06,438 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:01:06,691 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:06,929 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:01:06,929 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:06,929 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:01:06,930 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:01:06,931 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:01:07,133 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:01:07,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:08,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:09,135 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:01:09,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:10,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:11,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:11,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:12,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:13,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:14,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:15,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:16,312 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:01:16,313 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:01:16,313 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:01:16,315 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:01:16,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:17,142 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:01:17,143 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:01:17,315 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:17,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:18,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:19,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:20,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:22,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:25,437 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:01:25,437 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:25,438 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:01:26,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:27,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:27,817 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:01:27,817 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:01:27,818 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:01:27,819 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:01:28,152 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:01:28,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:28,820 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:29,153 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:01:29,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:30,085 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:01:30,437 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:31,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:32,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:33,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:34,437 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:34,437 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:37,681 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:38,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:39,590 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:39,636 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:01:39,637 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:01:39,637 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:01:39,639 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:01:39,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:40,164 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:01:40,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:01:40,436 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:01:40,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:41,165 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:01:41,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:42,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:44,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:47,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:48,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:49,312 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:01:49,313 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:01:49,314 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:01:49,315 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:01:49,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:50,173 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:01:50,315 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:50,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:51,173 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:01:51,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:52,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:53,334 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:55,681 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:01:56,033 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:01:56,034 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:01:56,080 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:57,033 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:58,253 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:01:58,253 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:01:58,254 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:01:58,254 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:01:58,255 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:01:59,180 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:01:59,181 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:01:59,212 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:00,086 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:02:00,212 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:01,088 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:01,213 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:04,618 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:05,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:06,619 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:06,725 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:06,725 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:02:06,727 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:02:06,727 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:02:06,728 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:02:07,188 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:02:07,681 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:09,190 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:02:11,729 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:15,505 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:02:15,505 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:02:15,548 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:16,504 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:16,745 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:17,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:02:17,437 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:02:17,437 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:02:17,439 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:02:17,504 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:18,199 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:02:18,504 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:19,200 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:02:19,504 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:20,505 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:21,505 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:22,505 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:22,505 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:25,374 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:26,373 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:27,557 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:27,605 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:27,605 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:02:27,606 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:02:27,606 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:02:27,607 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:02:28,211 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:02:28,556 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:29,212 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:02:29,557 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:30,088 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:02:33,090 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:36,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:36,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:02:36,588 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:02:37,586 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:38,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:38,708 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:38,709 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:02:38,709 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:02:38,710 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:02:38,711 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:02:39,222 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:02:39,665 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:40,665 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:41,225 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:02:41,665 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:42,665 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:43,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:44,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:44,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:45,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:46,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:47,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:48,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:49,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:49,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:50,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:51,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:02:51,588 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:02:51,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:52,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:53,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:54,666 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:54,896 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:02:55,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:56,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:57,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:58,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:02:59,126 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:02:59,126 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:02:59,127 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:02:59,128 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:02:59,244 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:02:59,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:00,090 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:03:00,092 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:00,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:01,246 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:03:01,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:02,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:03,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:04,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:05,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:05,667 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:06,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:03:06,588 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:03:06,668 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:08,489 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:09,489 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:10,685 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:10,728 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:10,729 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:03:10,729 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:03:10,730 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:03:10,731 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:03:11,261 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:03:11,684 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:12,684 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:13,263 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:03:16,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:20,655 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:21,587 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:03:21,588 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:03:21,654 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:21,892 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:22,625 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:03:22,625 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:03:22,626 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:03:22,627 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:03:22,654 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:23,276 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:03:23,654 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:24,654 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:25,278 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:03:27,628 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:30,092 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:03:30,266 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:31,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:31,984 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:03:31,985 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:03:31,986 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:03:31,987 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:03:32,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:32,285 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:03:32,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:33,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:33,286 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:03:34,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:35,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:38,266 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:39,272 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:03:39,273 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:03:39,316 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:40,272 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:41,193 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:03:41,193 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:03:41,194 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:03:41,195 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:03:41,272 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:41,294 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:03:42,272 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:43,273 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:43,296 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:03:44,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:44,273 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:45,272 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:46,273 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:47,273 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:48,947 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:49,266 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:49,947 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:50,928 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:03:50,929 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:03:50,929 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:03:50,931 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:03:50,947 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:51,305 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:03:51,947 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:53,307 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:03:54,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:03:56,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:56,937 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:03:56,938 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:03:57,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:58,893 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:03:58,893 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:03:58,894 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:03:58,895 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:03:58,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:03:59,316 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:03:59,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:00,093 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:04:00,095 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:01,318 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:05,095 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:06,358 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:07,356 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:08,237 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:04:08,237 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:04:08,238 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:04:08,239 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:04:08,325 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:04:08,356 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:09,326 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:09,356 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:10,239 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:10,356 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:15,203 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:15,203 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:04:15,204 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:04:15,821 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:16,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:17,117 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:04:17,117 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:04:17,117 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:04:17,119 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:04:17,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:17,334 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:04:18,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:19,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:19,337 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:20,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:21,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:24,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:25,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:26,843 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:26,888 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:04:26,889 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:04:26,889 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:04:26,890 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:04:26,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:27,344 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:04:27,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:28,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:29,346 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:29,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:30,094 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:04:30,203 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:04:30,203 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:04:32,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:34,096 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:35,096 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:35,882 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:04:35,882 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:04:35,883 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:04:35,884 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:04:36,096 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:36,354 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:04:37,096 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:37,355 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:37,884 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:42,861 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:43,861 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:43,862 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:44,992 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:44,993 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:04:44,993 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:04:44,994 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:04:44,995 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:04:45,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:04:45,203 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:04:45,363 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:04:45,945 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:47,365 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:49,449 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:49,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:50,938 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:51,938 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:52,180 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:04:52,181 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:04:52,181 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:04:52,183 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:04:52,371 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:04:52,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:53,372 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:04:53,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:54,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:04:54,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:55,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:56,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:04:59,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:00,096 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:05:00,099 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:00,202 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:05:00,203 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:05:00,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:01,845 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:05:01,845 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:01,845 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:05:01,846 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:05:01,847 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:05:02,379 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:05:02,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:03,380 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:05:03,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:04,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:05,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:05,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:06,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:07,822 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:10,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:11,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:12,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:13,213 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:05:13,213 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:13,214 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:05:13,214 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:05:13,215 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:05:13,386 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:05:14,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:15,387 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:05:16,216 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:19,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:19,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:05:19,076 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:05:20,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:21,000 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:05:21,001 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:05:21,002 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:05:21,003 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:05:21,074 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:21,392 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:05:22,003 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:22,074 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:23,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:23,393 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:05:24,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:25,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:26,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:27,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:27,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:28,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:30,097 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:05:31,437 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:32,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:32,436 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:33,628 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:05:33,629 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:33,629 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:05:33,630 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:05:33,631 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:05:34,074 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:05:34,075 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:05:34,402 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:05:34,588 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:35,403 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:05:37,703 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:37,704 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:38,703 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:39,536 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:05:39,537 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:05:39,538 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:05:39,539 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:05:39,703 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:40,407 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:05:40,703 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:41,408 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:05:43,539 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:48,541 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:48,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:49,074 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:05:49,075 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:05:49,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:50,685 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:05:50,685 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:50,685 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:05:50,686 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:05:50,687 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:05:51,418 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:05:51,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:52,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:53,420 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:05:53,650 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:53,650 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:54,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:55,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:56,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:58,650 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:05:58,673 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:05:59,673 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:00,099 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:06:00,813 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:06:00,813 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:00,813 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:06:00,814 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:06:00,815 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:06:01,428 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:06:01,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:02,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:03,430 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:06:03,673 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:03,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:04,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:06:04,075 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:06:09,322 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:13,983 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:14,983 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:14,983 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:16,241 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:06:16,242 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:16,242 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:06:16,242 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:06:16,243 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:06:16,443 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:06:17,241 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:17,444 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:06:18,241 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:19,075 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:06:19,075 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:06:19,241 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:20,241 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:20,300 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:25,300 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:27,391 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:28,390 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:29,701 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:29,701 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:06:29,702 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:06:29,702 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:06:29,703 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:06:30,101 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:06:30,459 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:06:30,657 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:31,102 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:31,459 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:06:31,657 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:36,103 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:38,696 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:38,697 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:06:38,697 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:06:39,695 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:40,752 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:40,753 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:06:40,753 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:06:40,754 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:06:40,755 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:06:41,470 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:06:41,708 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:41,755 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:42,708 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:43,472 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:06:43,708 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:47,696 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:52,697 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:56,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:56,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:06:56,933 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:06:57,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:58,149 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:06:59,060 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:06:59,061 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:06:59,061 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:06:59,062 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:06:59,063 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:06:59,489 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:07:00,014 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:00,101 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:07:01,014 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:01,491 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:07:02,014 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:03,014 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:03,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:08,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:11,016 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:11,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:07:11,932 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:07:12,015 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:12,925 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:07:12,925 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:07:12,925 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:07:12,927 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:07:13,015 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:13,505 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:07:14,015 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:14,927 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:15,507 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:07:19,928 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:24,546 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:25,546 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:25,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:26,554 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:26,772 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:07:26,773 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:07:26,773 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:07:26,774 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:07:26,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:07:26,932 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:07:27,524 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:07:27,554 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:28,554 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:29,526 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:07:29,554 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:30,103 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:07:30,554 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:31,104 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:35,810 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:36,810 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:36,812 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:37,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:37,903 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:07:37,904 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:07:37,905 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:07:37,906 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:07:38,538 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:07:38,843 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:39,539 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:07:41,907 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:45,572 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:07:45,573 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:07:45,613 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:46,572 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:47,592 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:47,636 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:47,637 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:07:47,637 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:07:47,638 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:07:47,639 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:07:48,551 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:07:48,591 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:49,552 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:07:49,591 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:52,640 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:55,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:56,329 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:57,353 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:07:57,353 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:57,353 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:07:57,354 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:07:57,355 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:07:57,562 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:07:58,329 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:07:58,355 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:07:59,564 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:00,105 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:08:04,107 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:04,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:08:04,376 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:08:04,420 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:05,375 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:06,148 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:08:06,149 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:08:06,149 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:08:06,151 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:08:06,375 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:06,572 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:08:07,375 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:07,573 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:08,375 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:09,151 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:09,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:10,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:11,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:12,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:13,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:14,181 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:14,182 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:08:14,183 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:08:14,183 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:08:14,185 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:08:14,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:14,582 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:08:15,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:15,584 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:19,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:22,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:08:22,456 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:08:22,500 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:23,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:24,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:24,711 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:24,945 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:08:24,945 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:08:24,946 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:08:24,947 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:08:25,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:25,594 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:08:26,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:27,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:27,596 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:28,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:29,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:30,106 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:08:30,108 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:30,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:32,836 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:33,836 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:34,957 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:08:34,957 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:34,958 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:08:34,958 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:08:34,959 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:08:35,605 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:08:35,911 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:35,959 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:36,911 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:37,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:08:37,455 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:08:37,607 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:37,911 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:41,836 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:42,450 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:43,451 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:44,733 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:44,733 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:08:44,734 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:08:44,734 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:08:44,736 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:08:45,616 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:08:45,689 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:46,689 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:47,618 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:47,690 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:47,736 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:52,837 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:08:53,368 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:53,369 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:08:53,369 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:08:54,368 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:55,349 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:08:55,349 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:08:55,350 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:08:55,351 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:08:55,368 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:55,627 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:08:56,368 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:57,368 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:08:57,630 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:08:58,352 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:00,108 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:09:03,147 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:04,147 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:04,149 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:05,149 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:09:05,149 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:05,149 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:09:05,150 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:09:05,151 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:09:05,640 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:09:06,147 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:07,642 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:09:09,152 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:13,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:13,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:09:13,457 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:09:14,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:14,690 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:15,477 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:09:15,477 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:15,477 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:09:15,478 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:09:15,479 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:09:15,651 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:09:16,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:17,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:17,653 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:09:18,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:19,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:20,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:20,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:21,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:22,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:23,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:24,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:25,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:26,433 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:26,476 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:09:26,477 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:26,477 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:09:26,478 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:09:26,479 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:09:26,664 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:09:27,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:27,665 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:09:28,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:09:28,456 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:09:28,497 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:29,460 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:30,109 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:09:30,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:31,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:32,110 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:32,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:33,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:34,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:35,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:36,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:36,785 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:09:36,785 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:09:36,786 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:09:36,787 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:09:37,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:37,676 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:09:37,787 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:38,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:39,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:39,678 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:09:40,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:41,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:42,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:43,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:09:43,456 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:09:43,501 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:43,695 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:44,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:45,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:46,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:46,849 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:09:46,849 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:09:46,850 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:09:46,851 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:09:47,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:47,687 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:09:48,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:49,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:49,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:49,689 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:09:50,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:51,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:54,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:09:58,947 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:09:58,947 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:09:58,947 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:09:59,946 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:00,111 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:10:00,112 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:01,053 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:01,055 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:10:01,056 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:10:01,056 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:10:01,057 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:10:01,703 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:10:02,053 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:03,706 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:10:06,058 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:11,058 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:16,059 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:16,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:10:16,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:16,523 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:10:17,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:18,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:18,892 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:10:18,893 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:10:18,893 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:10:18,895 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:10:19,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:19,725 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:10:20,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:21,727 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:10:21,895 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:25,774 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:26,773 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:27,774 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:27,877 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:27,877 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:10:27,877 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:10:27,878 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:10:27,879 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:10:28,736 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:10:28,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:29,737 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:10:29,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:30,112 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:10:30,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:31,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:10:31,522 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:10:31,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:32,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:33,748 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:33,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:34,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:35,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:36,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:37,995 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:38,774 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:38,995 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:40,125 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:40,125 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:10:40,127 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:10:40,128 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:10:40,129 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:10:40,749 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:10:41,084 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:41,750 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:10:42,084 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:43,084 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:44,774 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:48,969 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:10:48,969 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:10:49,016 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:49,968 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:50,229 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:51,137 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:10:51,137 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:51,137 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:10:51,138 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:10:51,139 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:10:51,762 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:10:52,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:53,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:53,764 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:10:54,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:55,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:55,969 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:10:56,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:57,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:58,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:10:59,352 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:10:59,353 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:10:59,353 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:10:59,354 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:10:59,771 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:10:59,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:00,114 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:11:00,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:01,115 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:01,774 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:11:01,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:02,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:03,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:03,968 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:11:03,968 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:11:04,918 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:06,261 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:08,446 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:09,446 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:10,446 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:10,821 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:11:10,821 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:11:10,822 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:11:10,823 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:11:11,446 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:11,786 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:11:11,823 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:13,787 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:11:16,824 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:19,555 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:19,555 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:11:19,555 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:11:20,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:21,433 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:11:21,433 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:11:21,434 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:11:21,435 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:11:21,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:21,796 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:11:22,435 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:22,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:23,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:23,798 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:11:24,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:27,553 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:29,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:30,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:30,115 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:11:31,120 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:11:31,121 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:31,121 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:11:31,121 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:11:31,123 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:11:31,809 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:11:32,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:33,076 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:33,123 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:33,811 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:11:37,170 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:37,171 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:11:37,171 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:11:38,170 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:38,402 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:39,193 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:11:39,193 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:39,193 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:11:39,194 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:11:39,195 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:11:39,819 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:11:40,170 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:41,170 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:41,821 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:11:44,195 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:45,359 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:46,359 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:47,682 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:11:47,682 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:11:47,682 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:11:47,683 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:47,684 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:11:47,829 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:11:48,681 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:49,684 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:49,831 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:11:54,685 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:54,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:54,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:11:54,785 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:11:55,783 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:56,833 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:11:56,833 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:56,833 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:11:56,834 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:11:56,835 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:11:56,840 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:11:57,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:58,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:59,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:11:59,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:11:59,843 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:12:00,118 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:12:00,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:01,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:02,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:03,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:04,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:04,790 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:05,791 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:07,029 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:07,029 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:12:07,029 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:12:07,030 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:12:07,031 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:12:07,853 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:12:07,985 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:08,985 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:09,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:09,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:12:09,784 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:12:09,856 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:12:09,985 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:10,985 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:11,985 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:12,985 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:13,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:14,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:14,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:15,840 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:12:15,841 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:12:15,842 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:12:15,843 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:12:15,862 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:12:15,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:16,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:17,864 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:12:17,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:18,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:19,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:19,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:20,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:21,986 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:22,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:23,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:24,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:12:24,785 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:12:24,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:25,001 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:25,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:26,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:27,485 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:12:27,486 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:12:27,486 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:12:27,487 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:12:27,878 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:12:27,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:28,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:29,881 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:12:29,987 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:30,118 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:12:30,119 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:31,038 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:31,988 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:32,988 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:35,120 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:35,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:36,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:37,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:37,929 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:12:37,929 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:12:37,930 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:12:37,931 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:12:38,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:38,893 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:12:39,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:39,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:12:39,784 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:12:39,895 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:12:40,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:40,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:41,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:42,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:43,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:44,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:45,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:45,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:46,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:47,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:47,716 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:12:47,717 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:12:47,718 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:12:47,719 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:12:47,907 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:12:48,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:49,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:49,910 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:12:50,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:50,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:51,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:52,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:53,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:54,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:54,784 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:12:54,784 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:12:55,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:12:55,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:56,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:57,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:58,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:59,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:12:59,876 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:12:59,877 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:12:59,878 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:12:59,879 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:12:59,920 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:13:00,120 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:13:00,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:01,456 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:01,458 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:01,922 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:13:02,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:03,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:04,459 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:06,457 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:10,072 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:13:10,073 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:13:10,112 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:11,071 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:12,132 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:12,176 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:13:12,177 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:12,177 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:13:12,178 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:13:12,179 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:13:12,933 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:13:13,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:13,934 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:13:17,180 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:20,161 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:21,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:22,061 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:13:22,061 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:13:22,062 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:13:22,063 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:13:22,159 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:22,943 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:13:23,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:23,160 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:23,945 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:13:28,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:30,122 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:13:33,124 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:36,831 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:36,831 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:13:36,831 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:13:37,829 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:38,680 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:38,845 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:38,845 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:13:38,846 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:13:38,846 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:13:38,847 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:13:38,961 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:13:39,844 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:40,844 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:41,844 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:41,964 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:13:42,844 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:43,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:44,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:45,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:46,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:46,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:13:46,937 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:13:46,938 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:13:46,939 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:13:46,970 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:13:47,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:48,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:48,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:49,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:49,975 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:13:50,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:51,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:51,830 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:13:51,830 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:13:54,114 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:55,522 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:56,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:57,545 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:13:57,545 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:57,545 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:13:57,546 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:13:57,548 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:13:57,987 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:13:58,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:59,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:13:59,549 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:13:59,989 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:00,122 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:14:00,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:01,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:02,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:03,521 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:04,893 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:04,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:05,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:06,929 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:14:06,929 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:14:06,929 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:14:06,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:07,155 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:14:07,155 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:14:07,157 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:14:07,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:07,998 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:14:08,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:09,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:10,001 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:10,157 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:10,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:11,932 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:12,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:13,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:14,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:15,893 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:15,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:16,933 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:18,088 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:18,089 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:14:18,089 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:14:18,090 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:14:18,091 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:14:19,012 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:14:19,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:20,013 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:21,092 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:26,092 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:27,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:14:27,617 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:14:27,657 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:28,615 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:29,757 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:14:29,757 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:29,757 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:14:29,758 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:14:29,759 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:14:30,026 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:14:30,124 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:14:30,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:31,125 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:31,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:32,028 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:32,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:33,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:34,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:35,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:36,617 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:37,255 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:38,254 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:39,341 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:39,341 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:14:39,341 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:14:39,342 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:14:39,343 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:14:40,038 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:14:40,299 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:41,299 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:42,040 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:42,343 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:44,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:44,470 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:14:44,470 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:14:45,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:46,444 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:14:46,445 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:14:46,445 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:14:46,447 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:14:46,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:47,046 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:14:47,447 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:47,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:48,047 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:48,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:52,448 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:54,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:55,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:56,373 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:14:56,374 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:14:56,375 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:14:56,376 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:14:56,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:57,058 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:14:57,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:58,059 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:14:58,376 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:14:58,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:14:59,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:14:59,469 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:14:59,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:00,126 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:15:00,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:01,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:02,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:03,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:04,128 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:04,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:05,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:05,836 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:15:05,837 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:15:05,837 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:15:05,839 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:15:06,068 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:15:06,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:07,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:08,070 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:15:08,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:09,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:09,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:10,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:11,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:12,512 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:14,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:14,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:15:14,469 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:15:14,715 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:15,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:16,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:16,440 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:15:16,440 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:15:16,440 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:15:16,441 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:15:17,081 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:15:17,082 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:15:17,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:18,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:19,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:20,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:20,511 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:21,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:22,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:23,438 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:25,312 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:26,311 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:26,312 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:27,489 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:27,489 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:15:27,489 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:15:27,490 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:15:27,491 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:15:28,095 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:15:28,441 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:29,441 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:29,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:15:29,469 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:15:30,097 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:15:30,127 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:15:30,441 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:31,441 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:32,312 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:32,441 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:33,656 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:33,657 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:15:33,657 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:15:33,658 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:15:33,659 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:15:34,103 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:15:34,615 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:35,615 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:36,105 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:15:36,615 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:38,313 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:42,164 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:43,163 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:44,106 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:44,148 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:15:44,149 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:15:44,149 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:15:44,150 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:15:44,163 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:44,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:15:44,469 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:15:45,116 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:15:45,163 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:46,118 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:15:46,163 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:49,695 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:52,492 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:53,492 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:54,492 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:54,828 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:15:54,868 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:15:54,869 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:15:54,869 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:15:54,871 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:15:55,130 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:15:55,492 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:56,492 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:15:58,134 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:15:59,871 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:00,129 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:16:01,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:01,102 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:16:01,102 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:16:02,100 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:02,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:02,742 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:02,743 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:02,744 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:16:03,100 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:03,143 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:16:04,100 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:05,100 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:05,146 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:16:05,745 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:06,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:09,046 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:10,046 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:11,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:11,105 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:11,105 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:11,106 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:11,106 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:11,107 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:16:11,156 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:16:12,063 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:13,063 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:13,159 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:16:14,063 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:15,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:16,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:16,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:16,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:16:16,101 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:16:17,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:18,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:19,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:20,065 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:20,373 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:20,374 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:20,374 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:20,375 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:16:21,065 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:21,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:21,171 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:16:21,172 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:16:22,065 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:23,065 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:26,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:28,365 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:29,364 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:30,131 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:16:30,509 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:30,509 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:30,510 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:30,510 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:30,512 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:16:31,101 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:16:31,101 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:16:31,191 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:16:31,191 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:16:31,298 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:31,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:32,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:33,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:34,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:35,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:36,365 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:36,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:37,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:38,468 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:39,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:40,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:41,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:41,860 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:41,905 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:41,906 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:41,906 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:41,907 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:16:42,209 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:16:42,469 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:43,211 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:16:46,908 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:49,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:49,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:16:49,771 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:16:50,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:51,616 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:51,616 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:51,616 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:51,618 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:16:51,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:52,228 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:16:52,618 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:52,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:53,231 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:16:57,600 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:58,600 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:58,601 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:16:59,685 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:16:59,685 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:16:59,686 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:16:59,686 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:16:59,687 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:00,132 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:17:00,249 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:00,640 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:01,251 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:01,640 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:02,640 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:03,640 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:04,601 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:04,641 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:04,769 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:17:04,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:17:05,641 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:06,641 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:07,005 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:17:07,005 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:17:07,006 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:17:07,007 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:07,262 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:07,641 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:08,641 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:09,266 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:09,601 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:14,602 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:16,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:17,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:18,049 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:17:18,049 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:18,050 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:17:18,050 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:17:18,051 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:18,282 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:19,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:19,283 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:19,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:17:19,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:17:20,035 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:20,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:21,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:22,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:23,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:24,044 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:24,785 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:17:24,785 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:17:24,786 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:17:24,787 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:25,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:25,294 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:25,787 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:26,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:27,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:27,298 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:28,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:29,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:30,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:30,136 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:17:31,137 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:33,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:34,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:34,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:17:34,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:17:35,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:35,268 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:17:35,268 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:17:35,269 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:17:35,270 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:35,310 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:36,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:36,270 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:37,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:37,313 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:40,495 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:41,495 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:41,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:42,329 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:17:42,329 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:17:42,329 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:17:42,331 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:42,495 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:43,322 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:43,323 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:43,495 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:44,495 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:45,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:46,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:47,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:47,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:48,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:49,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:49,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:17:49,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:17:50,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:53,325 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:53,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:17:54,326 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:55,265 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:17:55,265 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:17:55,266 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:17:55,267 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:17:55,326 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:55,340 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:17:56,326 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:17:57,343 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:17:58,496 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:00,136 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:18:02,584 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:03,583 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:03,585 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:04,429 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:18:04,429 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:18:04,430 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:18:04,431 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:18:04,583 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:04,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:18:04,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:18:05,357 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:18:05,358 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:18:05,583 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:06,583 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:07,584 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:09,583 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:10,674 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:11,674 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:12,768 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:18:12,769 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:12,769 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:18:12,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:18:12,771 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:18:13,372 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:18:13,721 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:14,771 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:15,375 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:18:17,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:18,386 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:19,128 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:18:19,129 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:18:19,129 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:18:19,130 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:18:19,381 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:18:19,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:19,770 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:18:19,770 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:18:20,040 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:20,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:21,385 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:18:21,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:22,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:23,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:24,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:25,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:25,722 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:26,387 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:27,388 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:28,388 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:29,388 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:29,724 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:18:29,725 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:18:29,725 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:18:29,727 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:18:30,137 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:18:30,388 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:30,396 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:18:31,138 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:31,388 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:31,398 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:18:32,388 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:36,722 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:38,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:38,197 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:18:38,197 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:18:39,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:40,225 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:18:40,225 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:40,225 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:18:40,226 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:18:40,227 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:18:40,409 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:18:41,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:41,410 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:18:42,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:42,227 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:43,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:44,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:45,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:48,050 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:48,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:49,050 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:50,120 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:50,121 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:18:50,121 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:18:50,122 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:18:50,123 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:18:50,421 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:18:51,079 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:51,422 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:18:52,080 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:53,080 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:53,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:18:53,196 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:18:53,392 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:54,080 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:55,080 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:58,393 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:18:58,432 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:18:59,432 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:00,140 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:19:00,609 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:19:00,609 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:00,610 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:19:00,610 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:19:00,611 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:19:01,436 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:19:01,437 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:19:01,566 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:03,612 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:07,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:08,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:08,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:19:08,196 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:19:09,070 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:09,071 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:19:09,071 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:19:09,072 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:19:09,073 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:19:09,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:09,447 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:19:10,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:11,093 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:11,450 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:19:12,094 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:13,094 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:14,094 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:15,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:16,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:17,217 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:19:17,217 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:17,218 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:19:17,218 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:19:17,219 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:19:17,458 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:19:18,204 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:19,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:19,220 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:19,460 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:19:20,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:21,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:22,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:23,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:19:23,196 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:19:23,236 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:24,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:24,425 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:25,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:26,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:27,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:28,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:29,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:30,141 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:19:30,143 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:30,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:31,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:32,205 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:33,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:34,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:35,171 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:35,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:36,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:37,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:38,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:19:38,196 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:19:38,237 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:39,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:40,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:40,455 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:41,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:41,713 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:19:41,713 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:19:41,714 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:19:41,715 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:19:42,206 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:42,490 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:19:43,207 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:43,491 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:19:46,171 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:50,098 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:51,097 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:52,099 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:19:52,145 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:52,145 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:19:52,145 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:19:52,146 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:19:52,147 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:19:52,505 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:19:53,102 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:53,196 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:19:53,196 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:19:53,506 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:19:54,102 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:55,102 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:19:58,097 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:00,143 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:20:02,319 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:03,318 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:03,320 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:04,425 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:20:04,425 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:04,425 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:20:04,426 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:20:04,427 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:20:04,530 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:20:05,383 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:05,531 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:20:06,383 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:08,428 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:13,428 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:18,048 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:18,048 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:20:18,049 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:20:19,047 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:19,330 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:20,154 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:20,157 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:20:20,157 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:20:20,157 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:20:20,159 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:20:20,551 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:20:21,154 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:21,553 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:20:22,154 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:23,154 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:25,047 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:30,047 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:30,144 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:20:35,146 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:35,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:20:35,910 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:20:35,956 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:36,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:38,009 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:38,010 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:20:38,010 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:20:38,011 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:20:38,012 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:20:38,575 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:20:38,967 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:39,576 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:20:39,967 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:40,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:40,967 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:41,967 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:42,967 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:43,968 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:44,968 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:45,968 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:46,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:46,968 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:48,089 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:49,089 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:50,233 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:50,233 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:20:50,233 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:20:50,234 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:20:50,235 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:20:50,593 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:20:50,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:20:50,910 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:20:51,188 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:51,595 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:20:52,188 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:52,532 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:20:53,188 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:54,188 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:55,188 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:20:57,533 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:00,146 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:21:03,147 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:05,650 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:05,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:21:05,909 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:21:06,649 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:07,865 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:07,865 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:21:07,866 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:21:07,867 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:21:07,868 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:21:08,622 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:21:08,819 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:08,869 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:09,623 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:21:09,819 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:10,819 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:11,819 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:12,819 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:14,650 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:16,401 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:17,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:18,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:18,889 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:21:18,889 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:21:18,890 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:21:18,891 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:21:19,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:19,638 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:21:19,891 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:20,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:20,909 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:21:20,910 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:21:21,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:21,641 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:21:22,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:23,400 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:24,401 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:25,401 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:25,401 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:27,849 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:28,849 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:29,957 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:21:29,957 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:29,957 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:21:29,958 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:21:29,959 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:21:30,148 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:21:30,656 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:21:30,911 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:31,149 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:31,657 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:21:31,912 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:32,911 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:36,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:36,741 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:21:36,741 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:21:37,016 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:37,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:38,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:39,061 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:21:39,062 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:21:39,062 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:21:39,063 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:21:39,670 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:21:39,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:40,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:41,673 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:21:42,064 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:47,065 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:47,889 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:48,889 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:49,981 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:21:49,981 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:49,982 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:21:49,982 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:21:49,983 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:21:50,687 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:21:50,938 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:51,689 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:21:51,740 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:21:51,740 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:21:51,938 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:52,889 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:52,938 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:53,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:54,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:55,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:56,941 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:57,260 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:21:57,261 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:21:57,261 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:21:57,263 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:21:57,699 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:21:57,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:58,889 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:21:58,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:21:59,702 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:21:59,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:00,150 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:22:00,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:01,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:02,939 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:03,890 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:03,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:04,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:05,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:06,836 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:22:06,837 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:22:06,837 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:22:06,838 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:22:06,839 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:22:06,839 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:22:06,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:07,715 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:22:07,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:08,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:09,718 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:22:09,890 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:09,940 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:14,891 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:15,936 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:16,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:17,859 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:22:17,860 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:22:17,860 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:22:17,861 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:22:17,934 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:18,734 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:22:18,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:19,735 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:22:19,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:20,862 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:20,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:21,794 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:22:21,795 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:22:21,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:22,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:23,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:24,935 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:26,045 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:26,176 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:22:26,177 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:26,177 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:22:26,178 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:22:26,179 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:22:26,747 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:22:27,133 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:27,748 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:22:28,133 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:30,151 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:22:31,153 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:31,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:32,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:33,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:33,500 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:22:33,501 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:22:33,502 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:22:33,503 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:22:33,758 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:22:34,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:35,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:35,761 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:22:36,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:36,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:36,794 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:22:36,795 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:22:37,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:38,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:39,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:40,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:22:40,333 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:40,333 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:22:40,334 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:22:40,335 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:22:40,770 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:22:41,187 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:41,287 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:41,772 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:22:42,287 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:46,188 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:48,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:49,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:50,329 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:22:50,330 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:22:50,330 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:22:50,331 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:22:50,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:50,787 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:22:51,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:51,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:22:51,789 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:22:51,794 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:22:51,795 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:22:52,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:53,333 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:54,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:55,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:56,332 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:22:56,333 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:23:00,155 DEBUG   SenderThread:1363837 [sender.py:send():379] send: stats
+2024-09-21 12:23:00,290 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:01,289 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:02,291 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:23:02,353 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:23:02,353 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:02,354 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:23:02,354 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:23:02,355 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:23:02,809 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:23:03,308 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:03,811 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:23:04,308 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:07,356 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:23:11,020 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:11,021 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: stop_status
+2024-09-21 12:23:11,021 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: stop_status
+2024-09-21 12:23:12,019 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:13,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:23:13,172 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:13,173 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:23:13,173 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:23:13,173 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:23:13,175 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:23:13,829 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:23:14,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:15,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:15,833 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/output.log
+2024-09-21 12:23:16,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:17,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:18,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:19,019 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:23:19,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:20,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:21,131 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:22,132 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:23,132 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:23,760 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: partial_history
+2024-09-21 12:23:23,761 DEBUG   SenderThread:1363837 [sender.py:send():379] send: history
+2024-09-21 12:23:23,762 DEBUG   SenderThread:1363837 [sender.py:send_request():406] send_request: summary_record
+2024-09-21 12:23:23,763 INFO    SenderThread:1363837 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end
+2024-09-21 12:23:23,847 INFO    Thread-12 :1363837 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_115259-p9bvnzls/files/wandb-summary.json
+2024-09-21 12:23:24,132 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages
+2024-09-21 12:23:24,763 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: status_report
+2024-09-21 12:23:25,132 DEBUG   HandlerThread:1363837 [handler.py:handle_request():158] handle_request: internal_messages