diff --git "a/wandb/run-20240921_111222-4ony16ec/logs/debug-internal.log" "b/wandb/run-20240921_111222-4ony16ec/logs/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/run-20240921_111222-4ony16ec/logs/debug-internal.log" @@ -0,0 +1,7902 @@ +2024-09-21 11:12:22,885 INFO StreamThr :2812040 [internal.py:wandb_internal():85] W&B internal server running at pid: 2812040, started at: 2024-09-21 11:12:22.884304 +2024-09-21 11:12:22,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status +2024-09-21 11:12:22,887 INFO WriterThread:2812040 [datastore.py:open_for_write():87] open: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/run-4ony16ec.wandb +2024-09-21 11:12:22,889 DEBUG SenderThread:2812040 [sender.py:send():379] send: header +2024-09-21 11:12:22,897 DEBUG SenderThread:2812040 [sender.py:send():379] send: run +2024-09-21 11:12:23,609 INFO SenderThread:2812040 [dir_watcher.py:__init__():211] watching files in: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files +2024-09-21 11:12:23,609 INFO SenderThread:2812040 [sender.py:_start_run_threads():1188] run started: 4ony16ec with start time 1726888342.884897 +2024-09-21 11:12:23,614 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: check_version +2024-09-21 11:12:23,614 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: check_version +2024-09-21 11:12:23,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: run_start +2024-09-21 11:12:23,872 DEBUG HandlerThread:2812040 [system_info.py:__init__():26] System info init +2024-09-21 11:12:23,872 DEBUG HandlerThread:2812040 [system_info.py:__init__():41] System info init done +2024-09-21 11:12:23,872 INFO HandlerThread:2812040 [system_monitor.py:start():194] Starting system monitor +2024-09-21 11:12:23,873 INFO SystemMonitor:2812040 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-09-21 11:12:23,873 INFO HandlerThread:2812040 [system_monitor.py:probe():214] Collecting system info +2024-09-21 11:12:23,873 INFO SystemMonitor:2812040 [interfaces.py:start():188] Started cpu monitoring +2024-09-21 11:12:23,874 INFO SystemMonitor:2812040 [interfaces.py:start():188] Started disk monitoring +2024-09-21 11:12:23,874 INFO SystemMonitor:2812040 [interfaces.py:start():188] Started gpu monitoring +2024-09-21 11:12:23,875 INFO SystemMonitor:2812040 [interfaces.py:start():188] Started memory monitoring +2024-09-21 11:12:23,877 INFO SystemMonitor:2812040 [interfaces.py:start():188] Started network monitoring +2024-09-21 11:12:23,935 DEBUG HandlerThread:2812040 [system_info.py:probe():152] Probing system +2024-09-21 11:12:23,937 DEBUG HandlerThread:2812040 [system_info.py:_probe_git():137] Probing git +2024-09-21 11:12:23,949 DEBUG HandlerThread:2812040 [system_info.py:_probe_git():145] Probing git done +2024-09-21 11:12:23,949 DEBUG HandlerThread:2812040 [system_info.py:probe():200] Probing system done +2024-09-21 11:12:23,949 DEBUG HandlerThread:2812040 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-1040-nvidia-x86_64-with-glibc2.35', 'python': '3.11.9', 'heartbeatAt': '2024-09-21T03:12:23.935425', 'startedAt': '2024-09-21T03:12:22.874534', 'docker': None, 'cuda': None, 'args': ('--local_rank=0', '--model_name_or_path', '/aifs4su/yaodong/models/Qwen2-VL-7B-Instruct', '--train_datasets', '/aifs4su/yaodong/datasets/aaa_dataset/TV2T-preference/extracted', '--output_dir', '../outputs/dpo_tiv2t_1.5k_base', '--train_data_files', '1.5k_clean_duplicate.json', '--train_template', 'NExTQA_preference', '--train_split', 'train', '--per_device_train_batch_size', '1', '--per_device_eval_batch_size', '1', '--gradient_accumulation_steps', '1'), 'state': 'running', 'program': '-m align_anything.trainers.tiv_to_t.dpo', 'codePathLocal': None, 'git': {'remote': 'git@github.com-hantao:htlou/align-anything.git', 'commit': '9eff99b0ee34f7265b8a2e1824650634c898f654'}, 'email': '2200017789@stu.pku.edu.cn', 'root': '/aifs4su/yaodong/projects/hantao/dev_cham/align-anything', 'host': 'dgx-119', 'username': 'yangyaodong', 'executable': '/aifs4su/yaodong/miniconda3/envs/hantao_tiv/bin/python3.11', 'cpu_count': 112, 'cpu_count_logical': 224, 'cpu_freq': {'current': 2064.1573660714284, 'min': 800.0, 'max': 3800.0}, 'cpu_freq_per_core': [{'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3799.996, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.003, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 3800.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}, {'current': 2000.0, 'min': 800.0, 'max': 3800.0}], 'disk': {'/': {'total': 1758.8549690246582, 'used': 14.712554931640625}}, 'gpu': 'NVIDIA H800', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}, {'name': 'NVIDIA H800', 'memory_total': 85520809984}], 'memory': {'total': 2015.5641632080078}} +2024-09-21 11:12:23,949 INFO HandlerThread:2812040 [system_monitor.py:probe():224] Finished collecting system info +2024-09-21 11:12:23,949 INFO HandlerThread:2812040 [system_monitor.py:probe():227] Publishing system info +2024-09-21 11:12:23,949 DEBUG HandlerThread:2812040 [system_info.py:_save_conda():209] Saving list of conda packages installed into the current environment +2024-09-21 11:12:24,610 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/conda-environment.yaml +2024-09-21 11:12:27,069 DEBUG HandlerThread:2812040 [system_info.py:_save_conda():224] Saving conda packages done +2024-09-21 11:12:27,072 INFO HandlerThread:2812040 [system_monitor.py:probe():229] Finished publishing system info +2024-09-21 11:12:27,080 DEBUG SenderThread:2812040 [sender.py:send():379] send: files +2024-09-21 11:12:27,080 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-09-21 11:12:27,362 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: python_packages +2024-09-21 11:12:27,363 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: python_packages +2024-09-21 11:12:27,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:27,365 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:12:27,365 DEBUG SenderThread:2812040 [sender.py:send():379] send: telemetry +2024-09-21 11:12:27,365 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:12:27,614 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/conda-environment.yaml +2024-09-21 11:12:27,614 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-metadata.json +2024-09-21 11:12:27,614 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/requirements.txt +2024-09-21 11:12:28,145 INFO wandb-upload_0:2812040 [upload_job.py:push():130] Uploaded file /tmp/tmpczc4l1tvwandb/hw2j0b1j-wandb-metadata.json +2024-09-21 11:12:28,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:28,615 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:12:28,615 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:29,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:29,617 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:30,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:31,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:32,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:33,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:33,620 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:34,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:34,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:12:35,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:36,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:37,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:37,621 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:12:37,621 DEBUG SenderThread:2812040 [sender.py:send():379] send: telemetry +2024-09-21 11:12:37,623 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:38,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:39,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:39,621 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:12:39,624 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:42,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:12:42,449 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:12:42,497 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:43,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:44,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:44,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:12:44,993 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:12:44,994 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:12:44,994 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:12:44,997 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:12:45,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:45,630 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:45,630 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_created():271] file/dir created: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:12:46,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:47,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:48,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:49,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:50,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:12:50,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:51,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:52,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:52,981 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:12:52,981 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:12:52,982 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:12:52,984 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:12:53,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:53,637 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:12:53,637 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:12:54,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:55,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:55,987 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:12:56,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:56,641 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/config.yaml +2024-09-21 11:12:57,448 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:12:57,449 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:12:57,489 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:58,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:12:59,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:00,451 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:01,451 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:01,640 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:01,685 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:01,686 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:01,686 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:01,688 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:02,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:02,646 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:03,648 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:07,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:07,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:08,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:08,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:08,597 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:08,598 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:08,600 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:08,653 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:09,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:09,654 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:10,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:11,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:12,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:12,448 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:13:12,449 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:13:12,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:13,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:15,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:16,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:16,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:16,929 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:16,930 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:16,932 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:17,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:17,661 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:17,662 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:18,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:18,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:19,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:20,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:21,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:22,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:23,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:23,626 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:23,877 DEBUG SystemMonitor:2812040 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-09-21 11:13:23,879 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:13:24,626 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:24,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:24,881 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:24,882 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:24,883 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:25,626 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:25,669 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:25,669 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:28,883 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:31,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:31,586 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:13:31,586 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:13:32,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:32,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:32,938 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:32,938 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:32,939 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:33,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:33,677 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:33,678 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:33,940 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:34,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:38,940 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:40,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:41,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:42,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:42,066 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:42,066 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:42,067 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:42,686 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:42,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:43,686 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:43,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:44,068 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:44,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:45,790 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:46,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:13:46,585 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:13:46,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:47,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:48,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:49,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:49,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:49,547 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:49,547 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:49,549 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:49,693 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:49,693 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:49,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:51,694 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:53,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:53,881 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:13:54,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:54,836 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:13:54,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:13:54,877 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:13:54,878 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:13:54,879 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:13:55,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:55,699 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:13:55,699 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:13:56,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:57,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:13:59,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:00,082 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:01,081 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:01,473 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:01,476 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:01,476 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:01,477 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:01,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:14:01,585 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:14:01,705 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:02,081 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:03,081 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:03,707 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:04,082 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:05,811 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:06,670 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:07,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:07,941 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:07,941 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:07,942 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:07,943 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:08,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:08,712 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:09,713 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:10,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:15,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:16,666 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:16,666 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:14:16,666 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:14:17,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:17,969 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:17,969 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:17,970 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:17,971 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:18,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:18,722 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:19,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:19,723 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:20,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:20,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:21,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:22,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:23,666 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:23,882 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:14:24,109 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:24,109 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:24,110 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:24,111 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:24,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:24,729 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:25,666 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:25,730 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:26,111 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:30,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:31,213 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:31,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:31,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:31,597 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:31,598 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:31,600 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:31,664 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:14:31,664 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:14:31,735 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:32,213 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:33,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:33,737 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:34,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:35,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:36,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:37,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:37,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:38,245 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:38,245 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:38,246 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:38,246 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:38,247 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:38,742 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:39,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:39,743 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:40,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:41,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:42,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:43,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:43,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:44,215 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:45,214 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:46,215 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:46,664 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:14:46,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:14:47,215 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:48,215 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:48,933 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:49,215 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:50,215 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:51,369 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:51,369 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:51,370 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:51,370 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:51,371 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:51,757 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:14:52,322 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:53,322 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:53,759 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:53,883 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:14:54,322 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:54,884 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:14:55,322 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:56,322 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:57,417 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:58,417 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:59,025 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:14:59,025 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:14:59,025 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:14:59,027 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:14:59,417 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:14:59,765 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:14:59,765 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:00,027 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:00,417 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:01,417 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:01,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:15:01,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:15:02,418 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:03,418 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:05,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:05,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:06,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:07,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:07,459 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:07,459 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:07,460 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:07,773 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:07,773 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:07,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:08,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:09,774 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:11,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:11,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:12,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:12,749 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:12,749 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:12,750 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:12,751 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:12,778 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:13,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:13,778 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:14,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:15,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:16,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:16,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:15:16,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:15:16,973 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:17,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:18,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:19,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:20,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:21,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:21,685 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:21,685 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:21,685 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:21,687 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:21,786 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:22,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:22,687 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:23,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:23,788 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:23,886 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:15:27,588 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:28,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:28,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:28,985 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:28,985 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:28,986 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:28,987 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:29,588 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:29,793 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:29,794 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:30,588 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:31,588 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:31,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:15:31,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:15:33,923 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:35,582 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:36,582 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:37,053 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:37,053 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:37,054 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:37,055 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:37,582 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:37,801 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:37,801 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:38,582 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:39,055 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:39,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:40,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:41,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:44,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:45,143 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:46,143 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:46,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:46,553 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:46,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:46,555 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:46,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:15:46,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:15:46,809 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:47,143 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:47,810 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:48,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:49,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:49,968 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:50,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:51,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:52,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:53,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:53,886 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:15:54,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:54,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:15:54,617 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:15:54,618 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:15:54,619 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:15:54,817 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:15:55,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:55,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:15:55,818 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:15:56,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:15:57,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:00,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:00,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:01,599 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:01,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:16:01,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:16:01,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:01,973 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:01,974 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:01,975 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:02,599 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:02,824 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:03,599 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:03,825 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:04,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:05,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:05,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:06,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:07,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:08,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:09,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:10,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:10,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:10,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:10,737 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:10,738 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:10,739 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:10,832 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:11,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:11,833 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:12,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:13,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:14,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:15,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:15,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:16,061 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:16,061 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:16,062 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:16,064 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:16,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:16:16,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:16:16,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:16,838 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:17,694 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:17,839 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:20,976 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:22,347 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:23,346 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:23,603 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:23,603 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:23,604 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:23,605 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:23,844 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:23,844 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:23,888 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:16:24,346 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:25,347 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:25,846 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:26,347 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:26,889 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:31,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:31,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:16:31,665 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:16:32,159 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:32,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:32,593 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:32,593 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:32,594 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:32,596 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:32,853 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:33,159 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:33,854 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:34,159 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:35,159 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:36,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:37,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:38,159 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:38,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:39,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:40,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:40,773 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:40,773 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:40,773 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:40,775 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:40,861 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:41,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:41,861 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:42,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:44,160 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:48,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:16:48,925 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:16:48,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:49,171 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:49,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:50,257 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:16:50,259 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:16:50,259 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:16:50,261 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:16:50,870 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:16:50,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:51,871 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:16:51,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:52,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:16:53,889 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:16:54,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:16:59,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:00,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:01,271 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:01,621 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:01,621 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:01,622 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:01,624 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:01,879 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:02,271 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:03,271 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:03,881 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:03,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:17:03,924 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:17:04,271 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:05,224 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:05,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:06,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:10,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:10,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:11,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:11,953 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:11,953 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:11,954 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:11,955 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:12,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:12,889 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:13,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:13,890 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:14,650 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:15,650 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:15,650 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:16,650 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:19,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:19,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:17:19,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:17:20,388 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:20,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:20,793 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:20,793 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:20,794 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:20,795 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:20,897 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:21,388 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:21,898 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:22,388 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:23,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:23,890 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:17:24,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:25,892 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:27,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:28,328 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:28,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:28,641 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:28,642 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:28,643 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:28,904 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:29,328 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:29,905 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:30,328 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:31,328 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:31,643 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:32,328 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:33,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:34,269 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:34,269 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:34,270 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:34,271 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:34,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:34,388 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:17:34,388 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:17:34,910 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:35,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:35,911 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:36,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:36,691 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:37,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:38,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:39,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:40,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:41,753 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:42,328 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:42,753 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:43,085 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:43,085 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:43,086 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:43,088 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:43,753 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:43,918 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:43,918 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:48,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:50,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:17:50,702 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:17:50,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:51,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:52,027 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:52,027 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:52,027 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:52,029 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:52,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:52,926 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:53,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:53,893 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:17:53,894 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:53,927 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:17:56,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:57,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:58,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:17:58,117 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:17:58,118 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:17:58,119 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:17:58,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:58,931 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:17:59,119 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:17:59,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:17:59,932 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:04,120 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:04,399 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:05,398 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:05,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:05,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:18:05,713 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:05,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:05,715 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:05,715 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:18:05,938 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:06,398 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:07,940 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:09,946 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:12,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:13,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:13,517 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:13,517 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:13,518 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:13,520 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:13,945 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:14,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:15,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:15,520 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:15,947 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:20,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:20,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:18:20,703 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:18:21,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:21,555 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:21,555 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:21,556 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:21,556 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:21,557 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:21,953 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:22,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:23,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:23,893 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:18:23,955 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:24,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:25,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:27,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:28,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:29,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:29,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:29,725 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:29,726 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:29,727 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:29,960 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:30,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:31,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:31,962 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:32,195 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:32,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:33,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:35,415 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:35,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:18:35,703 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:18:36,414 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:36,833 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:36,833 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:36,834 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:36,835 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:36,981 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:37,414 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:37,836 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:37,981 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:38,414 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:42,836 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:43,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:44,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:45,209 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:45,210 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:45,210 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:45,212 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:45,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:45,989 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:45,989 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:46,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:47,990 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:48,212 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:53,213 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:53,894 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:18:54,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:18:54,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:54,292 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:18:55,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:55,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:18:55,734 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:18:55,734 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:18:55,751 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:18:56,012 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:18:56,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:57,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:58,014 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:18:58,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:18:58,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:18:59,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:00,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:01,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:04,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:07,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:08,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:09,005 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:09,005 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:09,006 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:09,007 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:09,024 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:09,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:19:09,291 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:19:09,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:09,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:10,025 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:10,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:11,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:12,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:14,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:18,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:19,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:19,375 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:19,376 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:19,376 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:19,378 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:20,034 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:20,034 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:20,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:20,378 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:21,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:22,036 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:22,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:23,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:23,897 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:19:24,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:24,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:19:24,290 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:19:25,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:25,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:26,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:27,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:27,777 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:27,777 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:27,778 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:27,779 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:28,042 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:28,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:29,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:30,043 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:30,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:31,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:31,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:32,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:33,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:34,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:35,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:35,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:35,781 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:35,782 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:35,799 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:36,049 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:36,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:36,799 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:37,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:38,051 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:38,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:42,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:42,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:19:42,424 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:19:42,465 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:43,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:43,833 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:43,833 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:43,834 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:43,835 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:44,056 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:44,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:45,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:46,058 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:46,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:47,835 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:49,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:50,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:51,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:51,382 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:51,382 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:51,384 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:51,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:52,063 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:52,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:53,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:53,897 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:19:53,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:54,065 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:19:54,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:57,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:57,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:19:57,424 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:19:58,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:19:58,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:19:58,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:19:58,521 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:19:58,522 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:19:58,523 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:19:59,069 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:19:59,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:00,070 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:03,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:05,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:06,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:07,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:07,013 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:07,014 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:07,015 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:07,076 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:07,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:08,076 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:08,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:09,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:09,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:11,085 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:12,084 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:12,417 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:12,417 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:12,418 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:12,419 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:12,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:20:12,424 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:20:13,081 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:13,084 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:14,083 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:14,084 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:14,735 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:17,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:18,636 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:19,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:19,001 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:19,002 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:19,029 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:19,095 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:19,636 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:20,029 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:20,095 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:20,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:21,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:22,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:23,898 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:20:25,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:25,899 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:26,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:26,849 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:26,849 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:26,850 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:26,851 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:27,102 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:27,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:20:27,424 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:20:27,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:28,103 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:28,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:29,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:30,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:31,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:31,645 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:33,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:34,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:34,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:34,403 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:34,403 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:34,404 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:35,110 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:35,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:36,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:36,123 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:37,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:37,405 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:38,118 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:39,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:40,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:41,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:42,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:20:42,425 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:20:42,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:42,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:43,325 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:43,325 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:43,326 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:43,327 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:43,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:44,131 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:44,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:46,133 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:47,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:48,118 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:48,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:48,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:48,545 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:48,546 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:48,547 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:49,136 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:49,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:50,137 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:50,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:51,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:52,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:53,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:20:53,900 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:20:54,947 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:55,946 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:56,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:20:56,313 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:20:56,314 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:20:56,315 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:20:56,947 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:20:57,166 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:20:57,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:20:57,425 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:20:58,167 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:20:58,730 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:02,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:03,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:03,369 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:03,369 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:03,370 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:03,371 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:04,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:04,173 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:04,371 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:05,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:06,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:06,189 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:07,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:08,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:09,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:10,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:10,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:15,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:20,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:20,539 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:20,539 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:21:20,540 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:21:21,538 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:21,977 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:21,977 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:21,978 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:21,979 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:22,203 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:22,538 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:23,901 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:21:24,205 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:25,902 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:27,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:28,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:28,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:28,797 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:28,798 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:28,799 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:29,210 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:29,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:30,210 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:30,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:31,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:31,800 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:32,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:34,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:35,538 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:21:35,539 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:21:35,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:36,149 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:36,149 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:36,150 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:36,151 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:36,216 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:36,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:37,152 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:37,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:38,218 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:38,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:42,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:43,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:44,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:45,173 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:45,173 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:45,174 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:45,192 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:45,225 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:45,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:46,226 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:48,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:48,789 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:49,788 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:50,109 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:50,109 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:50,110 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:50,125 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:50,230 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:50,538 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:21:50,539 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:21:52,232 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:53,815 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:21:53,904 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:21:56,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: keepalive +2024-09-21 11:21:56,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:57,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:57,472 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:21:57,473 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:21:57,473 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:21:57,474 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:21:58,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:58,254 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:21:58,255 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:21:59,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:21:59,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:00,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:00,256 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:01,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:03,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:04,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:04,833 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:04,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:04,877 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:04,878 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:04,879 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:05,276 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:05,538 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:22:05,539 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:22:05,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:06,277 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:06,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:07,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:08,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:09,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:10,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:10,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:12,467 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:13,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:13,901 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:13,901 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:13,902 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:13,903 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:14,284 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:14,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:15,467 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:15,903 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:16,300 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:20,904 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:20,951 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:22:20,952 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:22:20,993 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:21,951 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:22,301 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:22,302 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:22,302 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:22,303 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:22,304 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:22,951 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:23,903 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:22:23,951 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:24,306 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:25,905 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:29,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:30,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:30,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:30,633 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:30,634 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:30,635 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:31,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:31,313 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:31,635 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:32,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:32,314 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:35,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:35,951 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:22:35,952 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:22:36,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:37,236 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:37,301 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:37,301 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:37,302 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:37,303 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:37,319 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:37,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:38,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:39,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:40,321 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:40,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:41,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:42,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:46,311 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:47,311 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:47,693 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:47,694 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:47,694 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:47,695 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:48,311 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:48,329 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:48,696 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:49,311 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:50,330 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:52,056 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:22:52,056 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:22:52,097 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:53,055 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:53,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:22:53,513 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:22:53,514 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:22:53,515 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:22:53,905 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:22:53,906 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:22:54,055 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:54,335 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:22:55,056 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:22:56,337 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:22:58,906 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:02,582 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:03,581 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:03,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:04,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:04,041 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:04,041 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:04,043 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:04,344 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:23:04,581 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:05,581 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:06,346 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:23:06,581 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:07,055 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:23:07,056 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:23:09,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:14,282 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:14,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: keepalive +2024-09-21 11:23:14,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:15,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:16,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:16,221 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:16,222 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:16,223 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:16,355 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:23:16,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:17,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:18,356 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:23:18,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:19,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:19,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:20,831 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:23,908 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:23:24,909 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:25,314 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:23:25,314 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:25,314 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:23:26,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:26,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:26,601 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:26,601 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:26,603 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:27,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:27,365 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:23:28,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:28,366 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:23:29,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:30,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:30,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:31,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:32,729 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:33,729 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:34,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:34,094 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:34,094 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:34,095 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:34,373 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:23:34,729 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:35,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:36,374 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:23:40,314 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:40,699 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:23:40,700 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:23:40,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:41,699 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:42,109 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:42,110 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:42,110 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:42,111 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:42,380 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:23:42,699 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:43,699 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:44,382 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:23:46,112 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:48,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:49,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:50,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:50,313 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:50,314 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:50,315 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:50,388 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:23:50,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:51,315 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:51,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:52,390 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:23:52,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:53,891 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:53,907 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:23:54,890 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:56,909 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:23:57,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:57,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:23:57,384 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:23:58,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:58,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:23:58,745 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:23:58,746 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:23:58,747 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:23:59,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:23:59,397 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:00,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:00,398 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:01,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:02,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:02,891 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:04,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:05,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:05,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:24:05,758 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:24:05,758 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:24:05,759 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:24:06,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:06,416 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:07,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:08,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:08,418 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:08,759 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:09,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:10,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:11,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:12,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:12,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:24:12,466 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:24:13,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:14,170 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:14,217 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:24:14,217 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:24:14,217 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:24:14,219 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:24:14,424 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:14,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:16,426 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:19,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:21,762 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:22,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:23,113 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:24:23,113 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:24:23,113 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:24:23,115 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:24:23,433 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:23,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:23,910 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:24:24,434 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:24,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:24,911 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:29,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:32,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:32,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:24:32,292 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:24:33,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:34,169 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:24:34,169 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:24:34,170 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:24:34,171 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:24:34,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:34,443 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:35,171 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:35,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:36,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:36,445 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:40,172 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:45,172 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:45,711 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:46,711 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:47,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:24:47,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:24:47,446 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:24:47,711 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:47,767 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:24:47,767 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:24:47,783 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:24:48,456 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:48,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:49,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:50,458 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:50,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:24:50,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:51,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:52,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:53,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:53,910 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:24:54,008 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:24:54,009 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:24:54,009 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:24:54,011 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:24:54,462 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:24:54,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:55,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:24:56,464 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:24:56,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:00,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:01,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:01,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:02,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:02,341 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:02,342 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:02,357 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:02,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:25:02,401 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:25:02,470 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:02,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:03,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:04,472 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:04,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:05,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:07,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:08,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:09,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:10,257 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:10,258 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:10,258 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:10,260 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:10,493 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:10,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:11,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:12,495 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:12,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:13,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:13,742 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:14,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:15,498 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:17,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:17,573 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:25:17,573 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:25:18,573 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:18,895 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:19,245 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:19,246 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:19,246 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:19,247 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:19,532 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:19,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:20,533 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:20,573 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:21,573 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:23,911 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:25:23,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:25,323 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:26,323 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:26,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:26,698 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:26,698 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:26,699 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:27,323 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:27,539 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:28,540 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:29,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:34,194 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:25:34,194 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:34,194 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:25:35,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:35,454 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:35,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:35,566 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:35,566 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:35,567 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:36,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:36,553 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:37,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:38,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:38,555 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:39,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:41,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:44,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:45,650 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:46,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:46,177 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:46,178 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:46,180 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:46,562 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:46,650 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:47,180 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:48,564 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:50,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:50,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:25:50,702 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:25:51,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:52,073 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:25:52,074 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:25:52,074 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:25:52,075 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:25:52,569 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:25:52,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:53,076 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:53,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:53,914 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:25:54,571 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:25:58,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:25:58,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:25:59,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:00,197 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:00,198 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:00,198 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:00,199 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:00,589 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:00,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:01,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:02,591 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:02,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:04,200 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:06,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:06,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:26:06,154 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:26:07,153 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:07,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:07,585 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:07,586 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:07,587 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:07,625 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:08,153 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:09,153 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:09,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:10,153 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:10,628 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:11,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:12,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:13,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:14,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:14,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:14,227 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:14,227 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:14,228 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:14,633 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:15,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:15,229 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:16,224 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:16,635 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:21,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:21,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:21,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:26:21,809 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:26:22,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:23,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:23,166 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:23,166 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:23,168 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:23,641 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:23,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:23,914 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:26:24,642 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:24,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:25,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:26,915 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:28,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:29,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:29,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:29,642 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:29,642 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:29,657 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:29,663 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:30,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:31,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:32,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:32,657 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:32,665 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:33,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:34,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:35,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:36,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:36,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:26:36,807 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:26:38,042 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:38,080 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:39,080 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:39,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:39,445 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:39,446 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:39,447 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:39,672 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:40,080 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:41,080 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:42,080 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:42,675 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:43,080 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:43,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:44,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:45,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:45,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:45,625 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:45,626 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:45,627 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:45,678 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:46,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:47,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:48,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:48,681 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:49,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:49,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:51,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:51,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:26:51,808 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:26:52,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:52,885 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:26:52,886 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:26:52,886 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:26:52,887 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:26:53,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:53,686 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:26:53,915 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:26:54,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:54,687 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:26:54,916 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:26:55,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:26:59,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:00,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:00,505 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:00,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:00,899 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:00,899 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:00,900 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:01,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:01,694 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:02,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:02,695 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:27:03,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:04,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:05,660 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:06,504 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:06,660 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:06,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:27:06,808 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:27:07,043 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:07,043 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:07,043 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:07,045 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:07,660 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:07,700 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:08,701 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:27:12,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:14,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:15,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:16,305 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:16,305 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:16,306 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:16,308 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:16,716 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:16,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:17,308 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:17,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:18,718 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:27:18,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:19,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:20,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:21,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:27:21,808 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:27:21,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:22,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:23,131 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:23,917 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:27:23,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:24,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:25,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:25,545 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:25,546 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:25,547 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:25,724 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:25,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:26,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:27,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:28,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:28,727 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:27:28,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:29,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:33,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:34,691 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:35,691 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:36,027 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:36,027 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:36,027 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:36,029 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:36,691 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:36,734 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:36,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:27:36,808 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:27:37,735 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:27:39,026 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:41,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:42,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:43,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:43,525 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:43,526 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:43,527 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:43,741 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:43,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:44,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:44,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:45,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:46,744 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:27:46,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:47,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:48,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:49,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:52,067 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:52,068 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:27:52,068 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:27:53,066 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:53,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:27:53,470 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:27:53,470 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:27:53,472 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:27:53,752 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:27:53,918 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:27:54,066 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:27:55,919 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:27:56,756 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:00,920 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:05,920 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:07,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:28:07,517 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:28:07,557 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:08,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:08,909 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:08,910 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:08,910 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:08,911 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:09,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:09,770 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:10,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:10,771 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:11,911 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:12,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:13,935 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:14,217 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:14,217 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:14,217 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:14,219 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:14,776 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:14,935 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:16,780 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:17,219 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:21,759 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:22,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:28:22,515 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:28:22,758 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:22,796 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:23,245 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:23,246 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:23,246 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:23,247 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:23,758 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:23,788 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:23,789 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:23,920 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:28:27,922 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:29,356 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:30,355 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:30,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:30,714 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:30,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:30,715 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:30,795 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:31,355 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:31,795 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:32,356 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:33,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:37,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:37,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:28:37,516 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:28:38,230 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:38,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:38,705 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:38,706 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:38,707 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:38,802 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:39,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:39,707 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:39,803 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:40,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:41,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:42,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:43,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:44,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:45,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:45,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:46,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:47,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:47,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:47,445 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:47,446 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:47,447 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:47,810 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:48,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:49,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:49,812 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:50,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:50,448 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:28:51,232 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:52,231 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:53,763 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:53,763 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:28:53,764 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:28:54,079 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:28:54,763 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:55,081 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:28:55,081 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:28:55,082 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:28:55,083 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:28:55,763 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:28:55,818 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:28:55,818 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:28:56,084 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:01,084 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:01,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:02,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:02,544 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:02,544 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:02,545 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:02,546 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:02,824 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:03,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:03,825 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:04,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:05,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:06,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:06,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:08,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:08,763 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:29:08,763 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:29:09,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:10,097 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:10,098 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:10,098 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:10,099 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:10,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:10,833 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:11,834 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:12,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:15,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:16,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:16,645 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:16,645 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:16,645 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:16,647 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:16,839 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:17,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:17,647 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:17,839 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:18,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:19,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:20,291 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:22,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:22,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:23,763 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:29:23,763 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:29:23,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:24,032 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:29:24,217 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:24,217 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:24,218 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:24,219 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:24,842 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:24,846 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:25,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:25,847 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:26,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:27,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:27,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:28,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:29,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:30,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:31,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:32,803 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:32,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:32,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:32,882 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:32,882 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:32,883 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:33,838 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:33,855 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:33,855 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:34,838 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:37,884 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:39,136 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:39,136 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:29:39,136 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:29:40,135 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:40,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:40,543 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:40,543 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:40,544 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:40,861 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:41,135 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:41,862 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:43,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:45,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:46,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:47,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:47,050 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:47,050 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:47,051 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:47,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:47,867 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:47,868 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:49,052 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:53,265 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:53,923 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:29:54,135 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:29:54,135 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:29:54,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:54,459 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:29:54,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:29:54,653 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:29:54,654 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:29:54,655 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:29:54,874 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:29:55,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:29:55,875 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:29:59,656 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:02,988 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:03,987 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:04,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:30:04,382 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:30:04,382 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:30:04,383 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:30:04,884 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:30:04,987 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:05,384 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:05,885 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:30:05,987 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:06,988 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:07,988 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:08,988 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:09,135 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:30:09,135 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:30:10,430 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:12,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:13,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:13,429 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:30:13,430 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:30:13,430 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:30:13,431 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:30:13,894 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:30:14,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:15,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:15,431 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:15,895 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:30:20,432 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:23,924 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:30:25,925 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:27,408 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:27,408 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:30:27,408 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:30:28,406 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:28,774 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:30:28,774 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:30:28,774 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:30:28,776 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:30:28,906 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:30:29,406 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:29,907 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:30:30,407 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:31,406 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:31,776 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:32,407 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:35,484 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:36,484 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:36,876 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:36,917 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:30:36,917 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:30:36,918 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:30:36,919 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:30:37,484 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:37,916 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:30:37,916 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:30:38,484 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:39,484 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:42,408 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:43,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:30:43,867 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:30:43,909 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:44,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:45,201 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:30:45,201 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:30:45,202 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:30:45,203 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:30:45,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:45,924 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:30:45,924 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:30:46,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:47,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:48,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:48,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:49,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:50,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:51,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:52,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:30:52,241 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:30:52,242 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:30:52,243 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:30:52,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:52,931 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:30:53,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:53,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:53,926 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:30:53,932 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:30:54,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:55,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:56,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:57,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:58,866 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:30:58,867 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:30:58,909 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:30:59,181 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:30:59,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:00,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:00,677 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:00,678 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:00,679 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:00,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:00,939 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:01,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:01,940 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:02,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:03,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:04,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:09,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:10,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:10,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:11,085 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:11,085 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:11,085 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:11,087 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:11,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:11,949 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:11,949 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:12,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:13,679 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:13,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:31:13,867 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:31:14,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:15,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:15,901 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:16,249 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:16,249 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:16,250 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:16,251 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:16,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:16,954 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:17,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:17,955 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:18,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:21,252 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:23,926 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:31:25,240 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:26,240 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:26,527 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:26,569 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:26,569 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:26,570 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:26,571 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:26,964 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:27,240 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:27,965 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:28,240 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:28,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:31:28,867 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:31:29,240 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:32,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:32,688 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:33,687 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:34,037 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:34,038 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:34,038 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:34,040 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:34,687 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:34,971 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:35,687 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:35,972 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:38,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:41,734 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:42,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:43,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:43,505 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:43,505 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:43,506 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:43,507 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:43,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:43,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:31:43,867 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:31:43,979 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:44,733 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:45,734 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:45,981 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:46,734 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:48,734 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:50,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:51,783 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:52,167 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:31:52,167 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:31:52,167 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:31:52,168 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:31:52,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:52,988 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:31:53,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:53,928 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:31:53,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:31:53,989 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:31:54,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:55,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:56,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:57,784 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:31:59,734 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:03,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:32:03,855 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:32:03,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:04,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:05,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:05,237 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:05,237 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:05,238 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:05,240 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:05,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:06,000 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:06,000 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:06,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:07,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:08,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:09,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:10,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:10,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:11,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:12,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:12,709 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:12,710 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:12,711 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:12,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:13,008 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:13,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:14,008 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:14,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:15,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:16,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:16,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:17,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:19,628 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:19,628 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:32:19,629 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:32:20,627 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:21,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:21,089 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:21,090 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:21,091 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:21,627 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:22,016 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:22,016 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:22,092 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:23,931 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:32:26,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:27,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:27,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:27,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:27,521 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:27,522 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:27,523 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:28,021 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:28,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:29,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:30,023 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:30,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:31,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:32,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:33,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:33,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:34,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:34,627 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:32:34,628 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:32:35,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:35,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:35,882 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:35,882 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:35,884 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:36,029 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:36,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:37,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:38,031 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:38,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:38,242 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:39,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:40,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:41,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:42,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:43,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:43,888 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:43,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:43,929 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:43,930 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:43,932 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:44,037 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:44,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:45,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:46,039 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:48,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:49,953 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:49,953 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:32:49,953 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:32:50,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:51,269 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:51,269 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:51,270 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:51,271 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:51,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:52,045 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:32:52,045 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:52,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:53,931 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:32:54,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:32:57,200 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:58,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:32:58,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:32:58,566 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:32:58,566 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:32:58,567 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:32:59,051 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:32:59,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:00,052 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:00,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:00,568 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:01,199 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:02,200 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:04,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:04,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:33:04,952 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:33:05,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:05,835 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:05,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:05,882 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:05,882 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:05,884 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:06,058 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:06,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:07,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:08,060 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:08,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:09,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:10,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:11,200 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:11,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:12,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:13,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:13,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:13,944 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:13,945 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:13,946 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:14,066 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:14,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:15,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:16,068 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:16,947 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:20,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:33:20,491 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:33:20,537 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:21,489 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:21,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:21,897 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:21,898 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:21,899 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:22,073 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:22,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:22,899 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:23,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:23,934 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:33:24,075 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:24,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:25,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:26,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:27,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:28,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:28,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:29,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:29,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:29,618 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:29,618 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:29,619 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:30,081 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:30,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:31,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:32,083 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:32,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:33,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:34,490 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:38,297 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:38,297 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:33:38,297 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:33:39,296 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:39,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:39,717 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:39,718 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:39,718 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:39,721 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:40,090 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:40,297 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:41,297 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:42,092 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:44,722 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:45,870 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:46,870 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:47,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:47,286 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:47,286 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:47,287 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:47,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:48,098 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:33:48,099 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:48,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:49,870 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:50,288 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:50,870 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:53,934 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:33:53,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:33:54,000 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:33:54,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:54,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:55,298 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:33:55,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:33:55,758 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:33:55,758 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:33:55,759 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:33:55,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:56,106 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:33:56,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:33:58,108 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:00,870 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:01,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:02,724 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:03,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:03,105 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:03,106 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:03,107 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:03,112 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:03,724 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:04,113 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:04,724 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:05,724 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:06,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:06,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:08,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:08,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:34:08,999 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:34:09,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:09,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:09,457 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:09,458 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:09,459 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:10,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:10,119 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:11,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:11,459 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:12,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:12,121 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:16,133 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:16,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:17,133 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:17,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:17,598 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:17,598 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:17,599 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:18,127 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:18,133 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:20,129 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:22,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:23,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:23,935 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:34:23,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:34:23,999 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:34:24,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:25,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:25,034 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:25,034 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:25,035 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:25,164 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:25,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:26,165 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:26,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:27,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:28,036 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:28,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:31,034 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:32,034 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:32,399 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:32,400 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:32,400 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:32,401 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:33,034 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:33,173 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:33,402 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:34,174 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:38,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:39,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:34:39,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:39,535 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:34:40,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:41,269 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:41,269 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:41,270 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:41,271 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:41,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:42,182 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:42,183 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:42,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:43,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:44,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:44,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:45,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:46,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:47,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:48,429 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:48,430 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:48,430 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:48,431 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:48,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:49,219 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:49,431 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:49,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:50,220 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:34:50,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:51,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:53,938 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:34:54,939 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:34:55,993 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:55,993 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:34:55,993 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:34:56,991 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:57,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:34:57,513 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:34:57,514 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:34:57,515 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:34:57,991 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:58,227 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:34:58,991 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:34:59,992 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:00,229 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:00,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:00,992 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:01,992 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:02,992 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:04,646 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:05,646 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:05,992 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:06,397 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:35:06,398 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:35:06,398 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:35:06,399 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:35:06,646 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:07,236 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:35:07,646 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:08,237 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:11,400 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:12,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:12,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:35:12,423 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:35:13,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:13,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:35:13,821 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:35:13,822 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:35:13,823 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:35:14,243 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:35:14,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:15,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:16,260 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:16,824 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:21,824 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:22,456 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:23,456 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:23,938 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:35:23,949 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:35:23,949 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:35:23,950 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:35:23,951 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:35:24,267 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:35:24,456 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:25,456 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:26,284 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:26,456 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:26,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:27,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:35:27,423 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:35:27,465 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:31,810 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:32,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:32,810 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:33,161 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:35:33,162 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:35:33,162 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:35:33,164 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:35:33,291 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:35:33,810 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:34,292 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:34,810 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:38,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:39,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:40,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:41,301 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:35:41,302 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:35:41,302 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:35:41,304 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:35:41,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:42,299 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:42,300 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:35:42,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:35:42,423 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:35:42,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:43,654 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:46,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:47,973 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:48,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:35:48,278 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:35:48,278 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:35:48,280 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:35:48,318 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:35:48,973 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:49,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:49,973 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:50,320 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:35:50,973 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:51,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:52,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:53,941 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:35:53,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:54,942 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:35:54,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:35:59,942 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:01,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:36:01,782 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:36:01,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:02,782 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:03,146 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:03,147 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:03,147 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:03,149 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:03,379 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:03,782 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:04,380 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:05,149 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:09,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:10,860 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:10,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:11,209 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:11,209 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:11,210 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:11,211 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:11,386 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:11,860 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:12,387 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:12,860 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:13,860 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:14,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:16,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:17,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:17,248 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:36:17,248 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:36:18,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:18,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:18,654 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:18,654 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:18,656 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:19,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:19,394 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:20,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:20,395 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:21,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:22,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:22,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:23,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:23,941 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:36:24,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:25,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:26,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:27,153 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:27,153 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:27,154 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:27,155 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:27,402 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:27,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:28,155 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:28,403 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:28,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:29,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:30,758 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:33,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:36:33,116 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:36:33,157 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:33,399 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:34,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:34,537 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:34,537 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:34,538 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:34,539 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:35,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:35,424 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:36,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:36,425 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:37,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:38,758 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:41,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:42,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:42,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:42,641 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:42,642 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:42,643 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:43,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:43,432 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:44,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:44,433 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:44,644 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:45,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:46,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:47,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:48,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:36:48,116 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:36:48,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:49,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:50,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:50,415 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:51,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:52,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:52,557 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:36:52,557 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:36:52,558 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:36:52,560 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:36:53,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:53,455 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:36:53,942 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:36:54,227 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:54,456 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:36:55,943 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:36:58,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:36:59,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:00,061 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:00,062 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:00,062 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:00,064 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:00,477 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:00,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:01,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:01,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:02,479 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:02,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:06,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:08,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:08,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:37:08,353 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:37:09,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:09,837 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:09,837 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:09,838 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:09,853 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:10,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:10,500 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:11,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:11,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:12,502 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:14,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:15,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:16,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:16,013 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:16,014 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:16,015 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:16,545 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:16,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:17,016 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:17,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:18,546 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:18,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:19,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:20,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:21,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:22,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:22,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:23,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:37:23,352 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:37:23,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:23,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:37:24,372 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:24,372 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:24,372 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:24,373 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:24,551 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:24,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:25,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:26,553 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:26,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:27,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:28,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:28,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:29,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:30,578 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:31,578 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:32,578 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:33,185 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:33,185 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:33,186 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:33,187 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:33,561 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:33,578 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:34,188 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:34,562 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:38,664 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:38,664 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:37:38,664 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:37:39,662 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:39,933 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:40,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:40,049 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:40,049 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:40,051 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:40,567 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:40,662 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:41,662 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:42,569 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:42,662 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:43,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:44,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:45,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:47,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:48,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:48,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:48,401 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:48,402 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:48,402 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:48,576 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:49,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:50,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:50,577 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:37:51,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:51,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:52,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:53,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:53,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:37:53,663 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:37:53,971 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:37:54,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:55,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:56,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:56,297 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:37:56,297 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:37:56,298 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:37:56,299 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:37:56,584 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:37:57,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:37:57,299 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:37:58,585 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:02,300 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:02,958 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:03,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:04,325 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:04,326 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:04,326 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:04,327 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:04,594 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:04,958 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:05,958 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:06,596 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:07,327 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:12,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:38:12,070 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:38:12,117 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:12,356 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:13,069 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:13,373 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:13,374 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:13,374 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:13,375 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:13,602 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:14,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:14,603 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:15,069 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:16,069 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:17,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:18,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:18,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:19,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:20,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:20,981 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:20,982 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:20,982 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:20,983 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:21,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:21,611 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:22,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:22,612 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:23,947 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:38:23,948 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:25,067 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:26,067 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:26,265 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:26,265 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:26,265 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:26,268 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:26,629 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:27,067 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:27,069 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:38:27,070 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:38:28,067 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:28,631 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:29,067 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:29,321 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:31,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:32,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:32,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:32,858 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:32,858 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:32,859 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:33,483 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:33,637 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:34,483 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:34,638 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:34,860 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:35,483 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:38,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:39,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:39,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:39,449 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:39,449 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:39,465 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:39,644 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:40,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:40,465 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:40,645 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:42,646 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:45,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:47,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:47,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:38:47,937 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:38:48,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:49,325 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:49,326 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:49,326 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:49,328 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:49,668 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:49,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:50,669 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:38:50,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:51,331 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:51,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:52,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:53,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:53,947 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:38:54,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:55,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:38:55,513 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:38:55,514 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:38:55,515 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:38:55,688 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:38:55,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:56,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:38:56,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:38:58,691 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:01,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:01,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:02,615 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:02,977 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:02,977 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:39:02,978 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:02,978 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:02,979 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:02,980 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:39:03,615 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:03,697 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:04,615 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:04,698 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:05,615 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:06,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:06,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:07,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:08,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:09,453 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:09,454 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:09,454 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:09,455 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:09,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:09,703 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:10,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:11,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:11,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:12,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:12,705 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:13,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:14,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:15,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:16,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:17,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:17,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:17,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:39:17,936 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:39:18,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:19,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:20,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:21,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:22,425 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:22,425 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:22,426 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:22,427 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:22,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:22,714 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:23,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:23,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:23,948 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:39:24,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:24,716 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:25,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:26,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:27,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:28,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:28,618 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:29,813 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:29,813 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:29,814 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:29,814 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:29,833 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:30,736 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:30,771 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:31,771 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:32,738 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:32,772 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:32,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:39:32,936 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:39:33,772 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:34,158 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:36,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:37,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:37,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:37,737 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:37,738 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:37,739 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:37,743 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:38,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:39,739 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:40,745 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:41,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:42,668 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:43,061 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:43,061 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:43,062 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:43,063 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:43,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:43,749 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:44,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:44,750 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:45,064 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:45,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:46,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:47,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:47,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:39:47,936 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:39:48,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:49,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:50,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:50,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:51,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:52,057 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:52,057 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:52,058 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:52,059 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:52,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:52,757 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:53,670 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:53,951 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:39:54,759 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:39:55,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:39:57,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:58,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:39:59,154 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:39:59,154 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:39:59,155 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:39:59,156 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:39:59,763 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:39:59,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:00,764 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:00,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:01,157 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:01,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:05,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:40:05,819 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:40:05,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:06,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:07,079 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:07,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:07,221 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:07,222 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:07,223 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:07,770 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:07,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:08,771 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:08,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:09,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:10,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:11,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:12,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:15,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:16,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:16,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:16,726 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:16,726 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:16,727 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:16,779 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:17,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:18,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:18,728 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:18,781 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:19,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:20,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:20,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:40:20,819 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:40:21,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:22,316 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:23,420 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:23,951 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:40:23,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:24,420 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:24,845 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:24,845 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:24,846 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:24,847 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:25,420 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:25,804 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:26,420 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:26,805 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:27,420 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:29,820 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:30,751 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:31,751 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:32,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:32,065 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:32,066 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:32,067 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:32,751 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:32,811 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:33,751 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:34,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:34,813 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:35,068 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:35,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:35,819 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:40:35,819 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:40:36,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:37,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:38,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:39,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:40,127 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:40,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:40,913 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:40,913 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:40,914 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:40,915 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:41,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:41,820 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:42,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:42,821 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:43,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:44,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:45,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:45,870 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:46,349 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:46,350 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:46,350 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:46,351 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:46,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:46,826 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:47,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:48,828 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:51,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:40:52,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:52,588 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:40:52,588 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:40:53,586 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:53,885 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:40:53,885 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:40:53,886 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:40:53,887 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:40:53,953 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:40:54,586 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:54,834 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:40:55,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:56,586 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:40:56,836 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:40:56,954 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:01,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:02,474 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:02,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:02,933 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:02,933 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:02,934 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:02,935 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:03,474 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:03,843 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:04,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:04,844 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:41:05,474 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:06,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:07,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:07,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:41:07,587 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:41:07,814 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:08,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:09,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:10,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:11,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:11,242 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:11,242 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:11,244 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:11,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:11,851 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:12,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:12,852 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:41:13,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:13,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:18,475 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:19,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:20,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:20,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:20,651 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:20,651 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:20,653 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:20,859 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:21,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:21,860 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:41:22,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:22,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:41:22,588 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:41:23,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:23,854 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:23,954 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:41:28,955 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:30,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:31,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:31,969 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:31,970 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:31,970 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:31,971 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:32,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:32,870 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:33,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:33,972 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:34,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:34,873 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:41:35,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:36,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:39,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:40,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:41:40,832 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:41:40,873 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:41,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:42,373 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:42,373 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:42,374 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:42,375 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:42,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:42,894 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:43,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:44,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:44,896 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:41:45,375 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:45,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:46,832 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:49,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:50,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:50,816 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:50,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:50,862 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:50,862 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:50,863 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:50,903 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:51,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:52,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:52,905 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:41:53,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:53,955 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:41:54,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:55,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:41:57,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:41:57,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:41:57,433 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:58,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:41:58,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:41:58,765 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:41:58,766 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:41:58,767 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:41:58,911 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:41:59,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:00,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:00,912 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:01,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:01,767 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:02,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:03,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:04,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:04,102 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:04,102 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:04,103 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:04,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:04,917 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:05,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:06,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:06,919 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:07,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:07,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:08,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:09,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:10,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:11,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:12,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:42:12,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:42:12,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:12,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:12,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:12,957 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:12,958 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:12,959 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:13,566 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:13,925 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:14,926 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:17,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:20,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:21,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:21,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:21,810 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:21,810 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:21,811 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:21,933 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:22,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:23,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:23,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:23,956 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:42:24,935 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:26,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:27,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:42:27,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:42:27,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:27,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:27,853 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:27,854 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:27,855 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:27,939 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:28,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:28,856 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:29,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:29,940 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:33,856 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:34,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:35,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:35,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:35,549 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:35,550 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:35,552 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:35,946 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:36,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:37,948 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:39,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:41,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:42,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:42:42,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:42:42,820 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:43,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:43,464 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:43,464 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:43,466 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:43,820 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:43,954 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:43,954 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:44,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:45,466 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:45,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:45,971 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:50,166 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:51,166 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:51,167 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:51,533 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:51,533 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:51,534 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:51,536 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:51,977 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:42:52,166 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:53,166 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:53,957 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:42:53,979 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:42:54,166 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:55,166 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:57,168 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:42:58,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:58,339 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:42:58,339 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:42:59,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:42:59,793 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:42:59,794 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:42:59,794 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:42:59,796 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:42:59,984 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:00,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:01,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:01,986 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:43:02,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:03,167 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:03,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:05,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:06,407 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:06,833 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:43:06,834 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:43:06,834 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:43:06,868 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:43:06,990 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:07,408 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:07,991 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:43:08,408 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:08,868 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:13,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:43:13,818 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:43:13,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:14,120 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:14,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:15,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:43:15,166 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:43:15,166 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:43:15,212 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:43:15,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:15,998 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:43:15,998 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:16,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:19,212 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:23,960 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:43:24,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:25,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:25,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:25,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:43:25,550 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:43:25,550 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:43:25,551 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:43:26,006 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:26,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:27,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:28,008 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:43:28,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:28,817 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:43:28,818 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:43:29,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:30,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:31,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:31,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:36,197 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:37,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:38,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:38,605 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:43:38,606 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:43:38,606 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:43:38,608 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:43:39,017 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:39,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:40,018 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:43:40,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:41,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:41,609 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:42,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:47,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:47,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:43:47,809 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:43:47,849 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:48,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:49,217 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:43:49,218 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:43:49,218 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:43:49,219 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:43:49,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:50,029 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:43:50,029 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:50,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:51,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:52,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:52,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:53,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:53,960 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:43:54,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:56,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:57,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:57,561 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:43:57,561 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:43:57,562 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:43:57,563 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:43:57,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:43:58,036 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:43:58,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:43:59,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:00,038 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:00,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:01,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:02,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:02,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:44:02,808 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:44:03,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:03,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:04,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:05,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:05,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:05,706 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:05,706 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:05,707 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:06,044 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:06,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:07,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:08,045 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:08,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:08,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:12,250 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:13,250 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:13,888 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:13,889 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:13,890 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:13,890 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:13,891 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:14,051 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:14,051 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:14,250 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:15,250 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:16,063 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:16,250 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:18,892 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:20,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:44:20,034 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:44:20,077 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:21,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:21,437 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:21,437 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:21,438 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:21,439 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:22,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:22,069 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:22,069 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:23,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:23,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:44:23,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:24,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:25,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:27,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:28,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:28,593 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:28,593 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:28,594 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:28,595 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:29,075 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:29,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:29,595 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:30,076 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:35,034 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:35,554 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:44:35,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:44:35,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:36,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:36,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:36,945 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:36,946 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:36,947 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:37,082 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:37,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:38,083 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:38,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:39,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:40,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:40,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:41,943 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:42,573 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:42,573 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:42,574 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:42,575 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:42,943 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:43,088 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:43,944 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:44,089 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:46,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:47,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:48,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:48,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:48,992 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:48,993 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:49,009 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:49,094 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:49,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:50,095 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:50,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:44:50,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:44:50,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:51,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:51,862 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:52,678 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:53,963 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:44:55,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:56,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:56,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:44:56,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:44:56,877 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:44:56,878 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:44:56,879 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:44:57,116 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:44:57,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:58,117 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:44:58,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:44:59,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:00,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:01,463 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:02,678 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:02,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:03,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:04,353 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:04,353 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:04,354 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:04,355 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:04,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:05,124 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:05,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:45:05,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:45:05,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:06,140 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:45:06,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:07,816 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:07,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:08,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:11,850 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:12,850 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:12,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:13,249 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:13,249 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:13,250 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:13,251 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:13,850 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:14,147 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:45:14,148 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:14,850 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:17,092 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:17,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:18,092 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:18,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:18,762 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:18,762 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:18,764 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:19,092 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:19,153 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:20,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:20,154 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:45:20,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:45:20,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:45:21,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:22,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:23,838 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:23,964 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:45:25,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:26,040 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:26,357 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:26,358 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:26,358 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:26,359 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:27,040 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:27,160 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:28,161 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:45:29,360 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:32,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:33,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:33,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:33,554 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:33,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:33,555 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:34,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:34,167 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:34,555 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:35,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:35,554 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:45:35,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:45:36,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:36,169 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:45:37,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:38,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:39,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:40,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:40,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:45,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:50,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:50,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:50,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:45:50,649 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:45:51,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:52,069 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:52,069 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:52,070 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:52,072 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:52,189 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:52,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:53,967 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:45:54,190 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:45:55,968 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:45:57,318 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:58,318 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:45:58,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:45:58,710 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:45:58,710 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:45:58,711 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:45:59,199 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:45:59,318 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:00,200 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:00,318 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:01,318 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:01,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:02,319 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:04,684 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:05,649 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:46:05,649 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:46:05,689 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:05,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:05,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:05,961 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:05,975 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:06,206 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:06,685 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:06,976 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:07,685 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:08,208 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:08,685 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:12,319 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:13,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:14,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:14,981 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:14,982 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:14,982 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:14,984 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:15,232 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:15,583 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:16,233 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:17,984 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:18,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:19,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:19,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:19,897 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:19,898 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:19,899 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:20,237 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:22,239 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:23,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:23,966 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:46:25,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:46:25,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:25,971 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:46:26,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:27,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:27,618 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:27,618 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:27,620 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:27,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:28,245 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:28,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:29,620 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:29,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:30,246 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:30,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:33,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:34,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:34,950 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:34,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:34,997 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:34,998 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:34,999 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:35,264 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:35,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:36,265 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:36,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:37,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:38,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:39,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:39,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:40,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:40,972 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:46:40,973 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:46:42,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:43,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:43,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:43,869 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:43,870 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:43,871 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:44,272 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:44,515 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:45,872 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:46,288 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:50,852 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:51,852 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:51,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:52,460 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:46:52,461 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:46:52,461 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:46:52,462 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:46:52,852 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:53,294 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:46:53,852 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:53,968 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:46:54,295 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:46:54,852 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:46:56,969 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:46:59,139 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:46:59,139 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:46:59,181 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:00,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:00,769 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:00,770 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:00,770 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:00,771 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:01,139 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:01,301 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:02,139 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:02,302 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:02,772 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:05,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:06,911 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:07,301 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:07,302 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:07,302 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:07,303 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:07,321 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:07,911 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:08,304 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:08,322 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:08,911 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:09,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:10,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:11,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:13,912 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:16,222 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:16,223 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:47:16,223 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:47:17,222 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:17,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:17,566 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:17,566 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:17,567 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:18,223 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:18,331 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:19,222 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:19,568 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:20,333 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:21,883 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:22,883 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:23,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:23,329 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:23,330 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:23,331 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:23,335 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:23,883 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:23,969 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:47:24,336 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:24,883 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:24,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:29,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:32,219 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:32,219 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:47:32,219 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:47:33,218 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:33,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:33,626 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:33,626 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:33,627 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:34,218 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:34,375 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:35,218 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:35,627 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:36,218 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:36,392 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:40,628 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:44,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:45,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:45,569 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:45,570 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:45,570 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:45,571 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:46,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:46,432 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:46,571 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:47,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:47,217 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:47:47,218 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:47:48,163 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:48,434 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:49,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:50,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:52,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:53,971 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:47:54,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:55,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:56,021 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:47:56,022 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:47:56,022 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:47:56,023 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:47:56,442 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:47:56,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:57,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:58,024 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:47:58,443 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:47:58,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:47:59,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:00,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:02,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:02,218 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:48:02,218 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:48:03,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:03,338 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:03,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:03,381 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:03,381 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:03,383 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:03,448 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:04,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:04,449 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:08,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:10,026 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:11,026 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:11,429 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:11,430 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:11,430 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:11,431 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:11,455 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:12,025 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:12,456 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:13,026 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:14,432 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:18,176 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:48:18,176 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:18,176 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:48:19,175 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:19,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:19,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:19,574 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:19,575 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:19,576 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:20,175 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:20,462 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:20,463 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:21,175 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:22,175 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:22,464 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:23,176 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:23,973 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:48:24,176 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:24,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:26,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:27,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:27,933 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:27,933 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:27,934 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:27,935 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:28,469 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:28,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:29,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:30,176 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:30,471 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:30,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:31,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:33,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:33,362 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:48:33,362 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:48:34,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:34,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:34,809 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:34,810 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:34,811 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:35,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:35,492 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:35,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:36,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:36,492 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:37,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:38,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:39,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:40,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:41,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:41,393 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:42,393 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:42,913 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:42,913 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:42,914 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:42,915 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:43,393 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:43,498 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:44,499 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:46,362 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:50,434 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:50,434 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:48:50,434 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:48:51,433 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:51,704 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:48:51,817 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:48:51,817 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:48:51,818 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:48:51,819 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:48:52,433 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:52,507 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:48:53,433 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:48:53,973 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:48:54,508 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:48:56,974 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:00,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:01,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:02,111 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:02,153 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:02,154 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:02,154 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:02,169 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:02,516 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:02,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:03,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:04,517 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:07,169 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:12,170 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:13,517 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:13,517 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:49:13,518 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:49:14,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:14,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:14,929 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:14,930 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:14,931 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:15,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:15,526 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:16,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:16,527 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:17,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:17,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:18,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:19,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:20,517 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:21,485 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:21,485 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:21,486 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:21,487 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:21,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:21,532 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:22,517 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:22,533 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:23,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:23,974 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:49:28,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:28,517 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:49:28,517 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:49:28,747 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:29,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:29,721 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:29,722 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:29,722 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:29,737 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:30,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:30,541 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:31,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:32,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:32,542 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:33,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:34,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:35,796 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:36,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:37,241 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:37,242 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:37,242 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:37,260 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:37,561 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:37,796 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:38,562 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:38,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:39,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:39,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:40,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:41,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:42,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:44,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:44,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:49:44,541 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:49:44,768 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:45,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:45,845 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:45,845 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:45,846 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:45,847 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:46,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:46,570 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:46,570 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:47,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:48,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:48,572 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:50,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:49:51,923 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:52,923 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:53,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:49:53,318 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:49:53,318 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:49:53,320 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:49:53,577 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:49:53,923 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:49:53,976 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:49:54,578 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:49:55,977 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:00,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:50:00,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:00,806 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:50:01,057 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:01,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:02,181 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:02,182 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:02,182 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:02,184 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:02,585 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:02,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:03,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:04,587 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:04,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:05,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:06,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:11,130 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:12,130 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:12,131 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:12,489 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:12,490 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:12,490 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:12,491 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:12,594 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:13,130 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:14,596 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:16,969 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:16,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:50:16,970 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:50:17,230 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:17,969 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:18,357 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:18,358 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:18,358 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:18,359 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:18,600 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:18,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:19,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:20,602 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:22,360 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:23,977 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:50:25,139 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:26,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:26,489 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:26,489 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:26,490 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:26,491 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:26,608 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:27,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:27,491 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:28,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:28,610 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:29,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:32,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:50:32,469 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:50:32,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:32,777 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:33,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:33,837 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:33,838 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:33,838 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:33,840 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:34,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:34,616 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:35,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:36,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:36,618 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:37,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:38,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:38,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:39,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:40,441 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:40,442 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:40,442 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:40,443 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:40,482 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:40,622 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:41,483 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:42,483 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:42,624 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:44,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:47,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:50:47,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:47,937 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:50:48,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:49,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:49,277 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:49,278 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:49,279 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:49,639 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:49,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:50,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:50,640 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:50,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:51,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:53,980 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:50:54,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:55,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:55,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:50:55,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:50:55,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:50:55,962 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:50:55,963 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:50:56,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:56,676 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:50:57,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:58,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:50:58,678 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:50:59,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:00,574 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:01,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:01,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:02,575 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:02,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:51:02,936 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:51:04,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:05,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:05,951 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:05,952 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:05,952 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:05,953 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:06,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:06,686 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:06,686 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:06,953 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:07,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:08,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:08,688 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:09,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:10,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:11,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:12,209 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:12,249 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:12,250 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:12,250 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:12,251 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:12,692 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:12,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:13,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:14,694 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:14,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:15,853 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:17,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:18,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:18,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:51:18,183 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:51:19,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:19,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:19,526 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:19,526 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:19,527 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:19,699 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:20,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:20,700 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:21,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:22,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:23,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:23,980 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:51:26,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:27,182 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:27,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:27,697 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:27,698 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:27,699 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:27,706 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:28,182 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:28,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:29,182 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:30,182 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:30,708 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:31,182 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:32,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:33,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:51:33,183 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:51:33,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:33,802 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:33,845 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:33,845 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:33,846 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:33,847 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:34,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:34,713 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:36,728 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:38,848 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:41,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:42,724 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:43,053 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:43,053 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:43,054 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:43,055 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:43,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:43,735 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:44,056 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:44,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:44,736 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:45,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:46,724 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:47,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:48,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:51:48,183 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:51:48,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:49,400 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:49,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:50,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:51,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:52,257 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:51:52,257 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:51:52,258 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:51:52,259 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:51:52,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:52,743 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:51:53,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:53,984 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:51:54,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:54,745 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:51:54,985 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:51:55,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:51:56,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:00,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:01,395 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:02,395 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:02,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:02,765 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:02,766 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:02,767 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:03,183 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:52:03,183 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:52:03,395 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:03,753 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:04,754 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:52:06,474 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:08,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:09,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:09,825 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:09,825 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:09,826 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:09,828 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:10,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:10,760 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:11,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:11,828 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:12,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:12,762 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:52:13,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:14,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:17,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:18,237 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:18,237 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:52:18,238 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:52:19,237 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:19,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:19,598 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:19,598 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:19,613 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:19,769 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:20,237 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:22,613 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:22,771 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:52:23,983 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:52:27,984 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:32,984 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:33,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:33,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:52:33,617 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:52:34,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:34,970 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:34,971 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:34,971 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:34,972 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:35,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:35,796 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:36,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:36,797 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:52:38,973 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:39,018 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:40,018 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:40,289 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:40,289 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:40,290 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:40,292 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:40,801 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:41,018 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:42,803 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:52:44,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:47,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:48,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:52:48,616 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:52:48,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:49,369 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:49,586 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:49,586 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:49,586 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:49,587 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:49,809 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:49,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:52,812 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:52:53,985 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:52:54,985 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:52:55,383 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:56,382 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:57,021 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:52:57,021 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:52:57,022 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:52:57,023 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:52:57,382 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:57,816 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:52:58,382 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:52:58,817 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:00,024 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:03,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:03,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:53:03,616 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:53:04,559 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:04,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:04,929 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:04,930 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:04,931 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:05,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:05,824 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:05,932 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:06,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:06,825 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:07,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:08,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:09,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:10,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:11,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:11,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:12,560 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:13,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:13,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:13,706 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:13,706 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:13,708 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:13,832 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:14,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:15,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:16,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:16,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:16,835 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:17,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:18,616 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:53:18,616 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:53:18,663 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:19,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:20,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:21,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:21,318 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:21,318 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:21,319 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:21,840 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:21,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:22,319 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:22,841 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:23,987 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:53:27,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:27,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:28,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:28,873 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:28,873 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:28,874 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:28,889 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:29,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:29,847 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:30,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:30,848 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:31,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:32,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:33,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:34,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:53:34,858 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:53:34,902 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:35,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:36,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:36,293 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:36,294 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:36,295 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:36,854 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:36,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:38,856 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:39,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:41,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:42,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:42,757 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:42,757 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:42,757 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:42,759 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:42,860 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:43,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:44,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:44,759 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:44,861 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:45,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:46,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:48,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:49,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:49,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:49,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:53:49,858 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:53:50,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:50,178 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:50,178 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:50,180 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:50,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:50,867 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:51,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:52,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:52,869 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:53:53,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:53,988 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:53:54,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:54,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:53:55,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:56,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:57,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:58,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:53:58,598 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:53:58,598 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:53:58,599 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:53:58,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:53:58,874 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:53:59,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:00,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:00,781 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:00,876 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:04,864 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:54:04,864 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:04,864 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:54:05,863 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:06,155 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:06,253 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:54:06,253 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:54:06,254 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:54:06,255 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:54:06,863 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:06,882 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:54:08,884 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:11,256 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:11,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:12,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:12,861 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:54:12,862 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:54:12,862 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:54:12,863 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:54:12,888 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:54:13,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:14,890 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:16,864 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:19,023 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:19,863 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:54:19,864 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:54:20,023 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:20,429 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:54:20,429 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:54:20,430 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:54:20,431 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:54:20,895 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:54:21,023 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:22,431 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:22,897 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:23,990 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:54:27,991 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:28,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:29,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:30,593 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:54:30,593 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:54:30,594 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:54:30,595 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:54:30,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:30,908 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:54:31,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:32,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:32,910 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:33,596 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:33,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:34,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:34,863 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:54:34,863 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:54:35,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:38,077 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:38,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:39,077 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:39,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:54:39,513 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:54:39,514 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:54:39,529 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:54:39,916 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:54:40,077 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:40,917 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:41,077 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:44,530 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:49,530 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:53,502 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:54:53,502 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:54:53,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:53,990 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:54:54,501 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:54,991 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:54:55,197 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:54:55,197 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:54:55,198 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:54:55,199 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:54:55,501 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:55,943 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:54:56,501 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:56,944 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:54:57,502 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:54:58,502 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:00,502 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:01,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:02,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:03,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:03,313 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:03,314 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:03,315 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:03,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:03,951 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:04,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:04,952 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:05,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:06,501 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:10,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:10,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:55:10,276 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:55:11,274 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:11,514 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:11,693 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:11,693 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:11,694 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:11,695 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:11,959 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:12,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:13,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:14,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:14,961 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:15,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:16,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:17,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:17,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:18,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:18,610 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:18,611 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:18,611 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:18,613 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:18,966 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:19,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:19,967 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:20,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:20,968 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:21,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:22,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:22,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:23,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:23,991 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:55:24,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:25,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:55:25,275 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:55:25,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:26,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:27,053 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:27,054 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:27,054 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:27,055 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:27,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:27,975 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:28,056 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:28,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:28,976 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:29,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:30,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:33,276 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:36,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:37,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:37,397 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:37,398 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:37,398 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:37,399 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:37,994 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:38,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:38,400 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:38,994 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:39,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:40,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:40,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:55:40,275 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:55:41,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:42,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:42,533 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:42,533 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:42,534 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:42,549 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:42,999 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:43,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:43,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:44,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:45,001 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:45,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:48,550 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:51,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:52,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:52,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:55:52,947 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:55:52,947 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:55:52,948 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:55:53,008 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:55:53,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:53,948 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:55:53,993 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:55:54,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:55,010 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:55:55,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:55:55,275 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:55:55,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:56,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:59,090 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:55:59,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:00,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:00,453 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:00,454 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:00,454 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:00,455 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:01,016 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:01,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:02,090 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:03,017 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:04,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:08,179 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:09,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:09,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:09,530 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:09,530 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:09,531 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:10,035 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:10,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:10,275 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:56:10,275 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:56:10,584 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:11,036 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:11,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:12,178 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:13,179 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:16,179 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:16,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:17,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:18,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:18,010 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:18,010 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:18,011 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:18,058 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:18,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:19,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:20,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:21,061 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:21,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:22,179 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:22,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:23,577 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:23,994 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:56:27,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:29,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:29,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:56:29,715 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:56:30,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:31,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:31,093 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:31,094 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:31,095 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:31,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:32,071 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:32,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:33,072 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:33,095 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:33,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:34,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:35,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:36,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:37,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:38,452 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:38,493 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:38,494 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:38,494 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:38,496 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:38,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:39,078 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:39,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:40,714 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:41,080 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:41,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:42,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:43,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:43,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:45,347 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:45,347 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:56:45,347 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:56:46,346 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:47,256 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:47,256 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:47,257 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:47,258 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:47,346 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:48,087 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:48,088 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:48,346 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:49,259 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:52,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:53,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:53,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:56:53,713 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:56:53,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:56:53,715 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:56:53,997 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:56:54,093 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:56:54,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:54,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:56:55,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:56,095 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:56:56,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:57,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:58,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:56:59,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:00,346 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:57:00,346 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:57:00,426 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:00,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:01,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:02,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:02,041 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:02,042 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:02,043 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:02,101 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:02,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:03,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:04,103 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:04,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:05,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:06,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:06,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:07,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:08,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:09,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:10,257 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:10,257 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:10,258 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:10,259 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:10,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:11,110 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:11,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:12,112 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:12,260 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:16,295 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:57:16,295 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:57:16,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:17,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:17,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:17,569 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:17,569 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:17,570 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:17,587 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:18,118 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:18,118 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:18,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:19,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:22,588 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:23,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:23,997 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:57:24,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:24,981 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:24,981 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:24,982 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:24,983 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:25,124 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:25,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:26,126 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:26,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:27,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:27,984 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:28,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:30,917 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:31,295 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:57:31,295 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:57:31,917 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:32,289 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:32,289 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:32,290 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:32,291 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:32,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:33,132 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:33,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:33,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:34,133 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:34,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:35,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:36,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:37,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:38,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:38,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:39,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:40,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:40,513 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:40,514 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:40,515 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:40,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:41,193 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:41,918 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:42,194 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:44,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:47,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:57:47,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:47,102 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:57:48,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:48,461 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:48,461 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:48,462 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:48,463 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:49,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:49,215 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:50,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:50,216 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:50,464 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:51,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:52,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:53,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:53,998 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:57:54,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:55,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:56,000 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:57:56,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:56,289 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:57:56,289 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:57:56,290 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:57:56,291 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:57:57,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:57,224 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:57:58,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:57:58,225 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:57:59,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:00,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:01,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:04,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:58:04,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:04,761 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:58:05,760 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:06,132 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:06,133 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:06,133 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:06,133 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:06,134 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:06,245 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:06,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:07,761 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:08,247 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:11,135 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:12,412 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:13,410 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:13,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:13,810 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:13,810 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:13,811 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:14,255 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:14,411 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:15,411 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:16,257 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:16,411 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:16,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:17,411 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:20,636 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:58:20,637 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:58:20,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:21,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:21,952 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:22,085 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:22,085 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:22,086 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:22,087 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:22,263 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:22,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:23,637 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:24,000 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:58:24,265 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:27,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:28,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:29,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:29,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:29,389 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:29,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:29,391 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:30,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:30,271 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:30,272 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:31,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:32,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:32,391 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:33,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:34,340 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:35,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:35,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:58:35,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:35,737 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:58:36,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:36,477 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:36,478 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:36,511 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:37,321 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:37,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:37,511 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:38,322 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:38,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:42,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:43,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:43,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:43,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:43,709 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:43,710 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:43,711 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:44,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:44,328 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:46,352 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:48,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:49,592 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:50,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:50,696 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:58:50,697 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:58:50,936 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:50,936 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:50,936 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:50,938 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:51,357 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:51,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:52,358 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:58:52,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:53,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:53,938 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:58:54,001 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:58:54,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:56,927 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:57,927 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:58,261 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:58:58,261 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:58:58,262 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:58:58,263 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:58:58,364 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:58:58,927 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:58:59,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:00,365 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:04,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:05,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:05,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:59:05,697 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:59:06,358 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:06,689 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:06,690 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:06,690 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:06,691 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:07,358 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:07,372 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:08,358 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:08,373 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:09,358 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:09,692 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:10,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:11,358 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:12,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:13,229 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:13,230 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:13,230 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:13,231 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:13,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:13,379 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:14,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:14,380 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:15,232 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:15,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:16,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:20,359 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:20,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:20,697 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:59:20,697 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:59:21,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:21,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:21,898 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:21,898 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:21,900 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:22,387 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:22,387 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:22,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:23,552 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:24,004 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:59:24,389 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:24,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:25,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:25,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:26,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:27,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:29,155 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:30,155 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:30,554 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:30,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:30,602 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:30,602 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:30,604 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:31,155 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:31,410 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:32,155 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:32,410 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:35,605 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:36,630 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:36,630 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:59:36,630 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:59:37,629 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:38,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:38,046 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:38,046 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:38,047 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:38,431 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:38,629 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:39,629 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:40,432 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:41,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:46,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:46,471 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:47,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:47,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:47,893 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:47,894 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:47,895 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:48,441 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:48,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:49,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:50,442 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:50,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:51,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:51,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:51,629 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 11:59:51,629 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 11:59:54,004 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 11:59:55,647 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:56,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 11:59:56,647 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:57,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 11:59:57,009 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 11:59:57,010 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 11:59:57,012 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 11:59:57,449 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 11:59:57,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 11:59:58,450 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 11:59:58,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:02,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:03,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:04,624 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:04,949 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:04,949 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:04,950 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:04,951 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:05,456 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:05,624 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:06,457 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:00:06,624 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:06,629 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:00:06,629 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:00:07,910 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:10,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:11,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:12,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:12,045 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:12,046 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:12,047 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:12,463 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:12,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:13,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:13,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:14,465 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:00:14,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:15,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:16,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:17,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:18,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:18,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:19,638 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:23,639 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:24,005 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:00:25,139 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:00:25,139 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:25,139 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:00:26,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:26,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:26,526 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:26,526 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:26,527 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:27,138 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:27,491 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:28,492 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:00:29,528 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:33,076 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:34,075 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:34,365 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:34,365 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:34,366 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:34,367 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:34,497 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:35,075 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:35,367 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:36,075 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:36,499 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:00:37,075 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:38,076 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:39,075 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:40,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:00:40,364 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:00:40,405 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:40,635 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:41,363 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:41,873 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:41,873 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:41,874 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:41,875 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:42,363 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:42,505 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:43,363 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:44,363 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:44,506 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:00:45,363 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:46,076 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:46,364 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:47,363 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:48,915 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:49,914 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:50,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:50,293 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:50,293 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:50,294 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:50,513 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:50,914 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:51,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:51,915 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:52,514 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:00:52,915 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:53,915 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:54,007 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:00:56,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:00:56,272 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:00:56,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:56,493 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:00:57,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:57,673 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:00:57,673 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:00:57,674 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:00:57,675 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:00:58,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:00:58,535 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:00:59,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:00,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:00,537 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:01,915 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:04,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:05,336 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:05,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:05,806 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:05,806 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:05,807 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:06,336 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:06,543 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:07,336 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:07,808 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:08,336 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:08,545 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:09,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:10,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:11,272 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:01:11,272 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:01:11,336 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:12,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:13,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:13,587 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:14,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:15,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:15,749 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:15,750 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:15,750 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:15,751 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:16,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:16,553 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:17,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:18,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:18,554 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:18,752 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:21,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:22,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:23,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:23,066 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:23,066 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:23,067 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:23,559 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:23,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:24,008 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:01:24,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:24,560 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:24,715 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:25,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:29,010 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:31,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:31,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:01:31,352 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:01:32,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:32,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:32,726 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:32,726 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:32,729 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:33,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:33,569 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:34,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:34,570 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:34,729 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:35,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:37,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:38,467 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:38,785 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:38,785 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:38,786 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:38,787 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:39,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:39,575 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:39,788 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:40,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:40,576 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:41,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:42,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:43,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:44,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:45,468 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:45,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:46,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:01:46,351 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:01:46,470 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:46,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:46,738 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:46,739 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:46,740 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:47,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:47,583 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:48,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:48,584 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:49,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:50,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:51,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:01:51,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:52,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:53,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:54,010 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:01:54,478 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:55,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:01:55,409 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:01:55,410 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:01:55,411 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:01:55,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:55,591 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:01:56,478 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:01:56,592 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:01:57,412 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:00,925 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:01,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:02:01,352 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:02:01,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:02,349 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:02,349 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:02,350 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:02,351 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:02,598 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:02,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:03,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:03,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:04,599 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:04,924 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:05,925 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:06,925 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:08,925 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:09,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:10,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:10,603 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:10,604 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:10,604 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:10,606 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:10,611 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:11,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:12,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:12,613 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:13,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:14,280 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:14,925 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:15,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:16,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:16,351 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:02:16,351 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:02:17,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:18,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:18,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:18,565 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:18,566 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:18,567 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:18,634 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:19,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:20,567 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:20,635 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:24,011 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:02:25,188 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:26,187 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:26,188 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:26,593 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:26,593 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:26,593 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:26,595 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:26,641 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:27,187 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:28,187 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:28,643 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:31,596 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:34,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:34,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:02:34,428 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:02:35,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:35,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:35,714 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:35,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:35,715 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:36,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:36,650 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:36,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:37,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:38,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:38,652 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:39,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:40,427 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:41,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:42,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:42,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:43,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:43,277 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:43,278 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:43,279 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:43,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:43,657 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:44,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:44,658 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:47,287 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:48,287 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:48,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:48,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:48,526 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:48,526 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:48,527 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:48,674 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:49,287 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:49,428 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:02:49,428 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:02:50,287 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:50,676 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:51,287 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:53,699 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:53,746 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:02:54,014 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:02:54,699 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:55,365 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:02:55,365 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:02:55,366 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:02:55,400 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:02:55,681 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:02:55,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:56,682 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:02:56,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:57,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:02:59,400 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:00,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:01,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:02,049 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:02,050 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:02,050 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:02,051 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:02,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:02,688 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:04,690 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:05,052 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:10,052 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:10,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:03:10,477 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:03:10,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:11,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:11,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:11,894 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:11,894 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:11,895 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:12,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:12,710 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:13,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:14,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:14,711 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:15,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:15,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:16,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:17,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:17,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:17,898 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:17,898 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:17,899 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:18,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:18,716 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:19,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:20,717 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:20,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:23,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:24,014 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:03:24,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:25,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:25,401 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:25,402 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:25,403 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:25,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:03:25,476 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:03:25,723 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:25,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:26,724 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:26,750 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:26,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:27,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:28,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:29,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:30,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:31,769 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:31,770 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:31,770 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:31,770 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:31,771 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:31,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:32,730 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:32,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:33,956 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:34,732 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:34,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:35,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:36,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:36,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:37,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:38,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:39,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:40,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:03:40,476 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:03:40,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:41,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:42,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:42,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:43,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:44,581 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:44,582 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:44,582 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:44,584 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:44,741 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:44,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:45,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:46,752 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:46,958 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:47,958 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:48,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:48,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:49,958 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:50,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:51,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:52,437 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:03:52,438 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:03:52,438 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:03:52,439 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:03:52,758 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:03:52,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:53,957 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:53,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:54,016 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:03:54,760 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:03:54,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:55,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:03:55,477 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:03:55,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:56,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:58,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:03:58,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:03:59,988 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:00,421 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:00,421 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:00,422 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:00,423 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:00,766 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:00,988 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:02,768 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:04,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:04,335 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:05,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:05,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:05,670 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:05,670 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:05,671 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:05,771 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:06,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:06,772 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:07,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:08,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:08,774 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:09,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:10,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:10,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:10,476 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:04:10,477 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:04:11,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:12,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:13,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:14,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:14,621 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:14,622 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:14,622 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:14,623 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:14,780 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:15,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:15,624 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:16,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:16,792 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:20,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:21,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:21,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:21,745 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:21,745 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:21,746 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:21,747 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:21,797 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:22,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:23,423 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:24,016 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:04:24,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:24,799 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:27,018 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:28,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:28,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:04:28,396 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:04:29,395 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:29,785 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:29,785 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:29,786 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:29,787 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:29,804 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:30,395 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:31,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:32,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:32,788 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:32,807 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:33,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:34,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:37,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:38,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:38,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:39,361 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:39,361 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:39,362 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:39,363 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:39,826 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:39,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:40,827 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:40,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:41,963 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:42,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:43,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:43,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:04:43,396 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:04:43,963 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:44,963 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:46,345 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:47,345 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:48,034 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:48,034 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:48,035 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:48,036 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:48,345 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:48,835 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:49,345 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:49,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:50,345 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:50,837 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:54,017 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:04:54,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:55,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:55,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:04:55,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:04:55,797 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:04:55,798 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:04:55,799 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:04:55,842 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:04:56,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:57,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:58,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:04:58,396 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:04:58,396 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:04:58,844 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:04:59,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:00,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:01,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:01,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:02,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:03,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:04,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:04,529 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:04,530 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:04,530 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:04,531 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:04,850 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:05,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:06,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:06,532 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:06,852 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:07,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:08,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:09,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:09,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:09,879 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:09,879 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:09,880 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:10,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:10,856 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:11,124 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:11,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:12,858 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:16,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:05:16,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:16,386 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:05:17,384 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:17,672 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:17,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:17,714 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:17,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:17,715 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:17,863 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:18,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:18,863 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:19,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:20,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:20,865 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:22,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:24,019 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:05:25,278 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:26,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:26,753 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:26,753 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:26,754 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:26,755 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:26,871 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:27,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:27,755 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:28,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:28,873 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:29,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:30,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:31,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:31,384 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:05:31,385 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:05:32,278 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:33,278 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:33,661 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:34,278 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:35,333 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:35,333 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:35,333 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:35,334 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:35,352 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:35,890 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:36,289 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:36,891 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:37,290 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:39,352 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:43,419 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:44,418 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:44,419 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:44,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:44,797 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:44,798 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:44,799 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:44,899 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:45,418 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:46,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:05:46,385 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:05:46,425 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:46,901 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:47,418 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:49,646 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:54,020 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:05:54,706 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:05:54,706 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:55,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:56,161 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:05:56,161 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:05:56,162 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:05:56,163 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:05:56,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:56,925 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:05:56,925 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:05:57,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:58,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:05:59,706 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:00,705 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:00,706 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:01,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:06:01,385 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:06:05,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:06,038 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:06,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:06,405 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:06,406 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:06,406 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:06,423 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:06,934 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:07,038 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:08,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:08,935 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:09,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:10,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:11,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:12,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:13,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:14,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:14,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:14,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:14,961 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:14,963 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:15,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:15,943 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:16,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:06:16,385 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:06:16,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:16,943 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:17,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:17,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:18,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:19,585 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:21,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:22,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:22,921 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:22,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:22,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:22,962 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:22,963 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:23,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:23,951 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:24,023 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:06:24,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:24,952 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:25,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:26,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:27,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:28,358 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:28,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:28,402 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:28,402 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:28,404 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:28,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:28,956 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:29,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:30,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:30,958 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:31,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:06:31,386 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:06:31,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:32,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:33,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:33,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:34,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:35,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:36,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:37,537 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:37,538 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:37,538 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:37,540 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:37,877 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:37,965 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:38,878 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:38,966 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:39,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:44,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:45,034 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:46,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:46,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:06:46,445 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:46,446 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:06:46,733 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:46,733 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:46,735 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:46,974 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:47,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:48,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:48,976 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:49,736 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:51,255 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:52,254 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:52,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:06:52,541 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:06:52,542 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:06:52,543 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:06:52,980 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:06:53,254 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:54,023 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:06:54,254 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:54,982 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:06:55,025 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:06:58,816 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:06:59,815 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:00,186 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:00,233 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:00,234 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:00,234 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:00,235 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:00,815 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:00,989 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:00,989 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:01,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:07:01,402 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:07:02,990 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:05,696 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:06,531 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: keepalive +2024-09-21 12:07:06,531 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:07,531 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:07,885 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:07,886 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:07,886 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:07,887 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:08,006 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:08,531 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:09,007 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:09,531 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:10,888 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:14,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:15,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:15,769 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:15,770 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:15,770 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:15,771 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:16,014 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:16,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:07:16,402 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:07:16,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:16,718 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:17,015 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:17,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:18,449 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:19,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:22,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:22,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:23,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:24,024 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:07:24,061 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:24,063 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:24,063 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:24,064 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:24,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:25,023 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:25,023 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:25,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:26,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:27,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:28,450 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:29,778 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:30,778 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:31,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:31,093 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:31,093 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:31,109 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:31,401 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:07:31,402 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:07:31,777 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:32,030 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:32,778 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:33,031 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:33,664 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:38,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:39,203 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:39,203 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:39,653 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:39,654 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:39,654 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:39,655 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:40,038 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:40,203 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:41,039 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:41,203 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:42,203 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:44,656 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:48,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:07:48,642 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:07:48,685 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:49,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:49,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:50,085 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:07:50,085 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:07:50,086 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:07:50,087 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:07:50,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:51,048 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:07:51,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:52,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:53,050 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:07:53,642 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:54,026 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:07:54,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:07:55,027 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:07:55,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:00,028 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:01,731 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:02,730 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:03,093 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:03,094 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:03,094 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:03,096 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:03,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:08:03,641 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:08:03,730 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:04,076 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:04,730 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:05,077 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:05,730 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:05,856 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:06,731 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:11,732 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:12,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:13,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:13,569 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:13,570 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:13,570 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:13,571 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:14,086 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:14,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:15,087 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:15,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:16,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:17,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:17,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:18,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:18,641 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:08:18,641 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:08:19,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:20,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:21,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:21,905 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:21,905 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:21,906 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:21,907 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:22,094 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:22,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:22,540 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:23,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:24,028 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:08:24,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:25,097 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:25,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:26,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:27,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:28,184 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:28,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:29,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:29,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:29,930 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:29,930 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:29,931 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:30,102 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:30,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:31,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:32,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:33,104 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:33,185 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:36,604 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:08:36,604 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:08:36,645 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:37,603 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:37,993 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:37,993 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:37,994 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:37,996 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:38,109 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:38,604 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:38,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:39,604 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:40,603 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:41,112 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:43,966 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:44,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:44,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:45,317 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:45,318 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:45,318 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:45,320 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:45,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:46,117 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:46,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:47,118 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:47,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:50,320 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:51,026 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:51,604 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:08:51,604 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:08:52,025 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:52,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:08:52,478 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:08:52,478 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:08:52,479 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:08:53,025 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:08:53,125 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:08:54,031 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:08:55,126 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:08:56,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:08:59,448 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:00,447 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:00,756 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:00,757 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:00,757 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:00,758 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:01,132 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:01,447 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:01,759 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:02,133 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:02,448 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:03,448 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:04,806 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:05,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:06,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:06,065 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:06,066 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:06,067 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:06,149 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:06,604 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:09:06,604 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:09:06,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:06,834 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:07,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:09,152 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:11,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:12,246 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:12,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:12,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:12,617 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:12,618 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:12,619 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:13,156 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:13,246 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:14,246 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:15,171 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:15,246 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:16,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:18,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:18,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:19,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:20,433 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:20,434 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:20,434 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:20,435 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:20,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:21,177 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:21,604 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:09:21,604 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:09:21,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:22,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:23,179 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:23,855 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:23,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:24,031 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:09:24,994 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:26,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:27,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:27,945 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:27,945 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:27,946 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:27,947 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:28,184 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:28,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:28,948 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:29,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:30,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:31,187 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:31,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:32,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:34,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:35,344 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:36,344 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:36,793 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:09:36,793 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:36,794 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:09:36,978 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:36,978 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:36,979 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:37,193 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:37,344 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:38,344 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:39,195 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:39,980 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:40,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:41,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:41,997 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:41,998 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:41,998 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:41,999 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:42,198 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:42,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:43,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:44,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:45,000 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:45,201 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:45,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:48,260 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:49,259 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:49,617 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:49,617 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:49,618 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:49,619 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:50,206 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:50,259 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:50,619 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:51,206 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:54,033 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:09:55,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:09:55,293 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:09:55,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:56,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:56,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:09:56,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:09:56,806 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:09:56,806 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:09:56,807 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:09:57,212 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:09:57,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:58,292 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:09:59,214 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:09:59,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:00,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:01,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:02,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:02,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:03,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:04,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:07,294 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:12,295 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:13,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:10:13,797 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:10:13,841 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:14,796 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:15,149 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:15,149 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:15,150 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:15,195 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:15,228 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:15,796 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:17,230 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:10:18,196 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:20,509 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:21,509 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:21,795 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:21,796 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:21,796 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:21,798 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:22,235 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:22,509 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:23,509 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:23,798 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:24,033 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:10:24,236 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:10:24,509 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:25,510 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:27,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:28,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:28,796 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:10:28,796 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:10:28,982 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:28,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:28,995 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:28,996 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:28,997 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:29,242 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:29,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:30,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:31,244 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:10:31,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:34,510 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:36,551 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:37,550 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:38,033 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:38,034 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:38,034 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:38,035 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:38,253 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:38,551 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:40,036 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:40,254 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:10:41,670 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:42,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:42,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:42,989 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:42,990 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:42,991 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:43,257 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:44,258 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:10:45,992 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:49,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:10:49,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:49,123 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:10:50,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:50,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:50,522 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:50,522 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:50,525 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:51,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:51,298 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:51,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:52,122 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:52,299 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:10:53,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:54,050 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:10:54,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:56,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:57,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:10:57,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:57,869 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:10:57,870 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:10:57,870 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:10:57,871 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:10:58,305 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:10:58,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:10:59,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:00,307 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:00,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:01,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:02,123 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:02,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:03,548 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:05,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:11:05,294 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:11:05,337 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:06,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:06,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:06,805 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:06,806 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:06,807 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:07,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:07,314 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:07,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:08,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:08,315 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:12,807 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:13,773 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:14,773 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:15,209 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:15,209 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:15,210 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:15,253 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:15,322 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:15,773 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:16,323 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:16,773 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:18,253 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:21,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:11:21,780 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:21,780 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:11:22,779 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:23,289 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:23,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:23,330 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:23,330 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:23,331 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:23,779 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:24,036 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:11:24,331 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:24,331 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:24,779 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:28,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:28,488 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:29,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:29,837 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:29,838 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:29,838 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:29,840 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:30,337 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:30,489 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:31,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:32,339 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:32,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:33,488 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:34,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:34,488 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:38,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:11:38,887 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:11:38,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:39,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:40,073 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:40,192 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:40,192 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:40,193 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:40,194 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:40,347 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:40,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:41,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:42,349 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:44,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:45,541 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:45,542 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:45,901 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:45,901 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:45,902 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:45,912 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:46,353 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:46,542 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:47,542 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:48,355 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:50,913 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:54,037 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:11:54,826 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:54,826 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:11:54,826 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:11:55,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:56,065 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:11:56,321 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:11:56,322 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:11:56,322 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:11:56,324 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:11:56,362 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:11:56,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:57,817 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:11:58,363 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:11:58,818 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:01,324 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:06,325 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:06,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:07,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:08,433 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:08,434 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:08,434 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:08,435 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:08,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:09,374 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:09,817 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:12:09,818 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:12:09,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:10,374 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:10,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:11,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:11,872 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:12,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:16,872 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:17,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:18,544 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:18,937 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:18,937 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:18,938 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:18,939 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:19,383 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:19,544 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:20,384 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:20,544 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:21,544 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:21,940 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:22,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:23,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:24,040 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:12:24,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:24,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:12:24,930 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:12:25,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:26,261 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:26,262 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:26,262 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:26,263 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:26,405 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:26,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:27,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:28,407 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:32,546 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:32,846 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:33,846 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:34,301 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:34,302 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:34,302 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:34,303 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:34,412 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:34,846 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:35,846 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:36,428 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:38,304 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:41,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:41,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:12:41,102 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:12:42,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:42,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:42,478 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:42,478 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:42,479 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:43,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:43,435 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:43,480 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:44,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:44,436 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:45,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:46,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:47,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:49,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:49,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:50,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:50,809 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:50,809 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:50,810 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:50,812 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:51,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:51,442 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:52,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:52,443 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:53,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:54,040 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:12:54,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:55,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:12:56,190 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:12:56,190 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:12:56,233 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:57,189 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:57,561 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:12:57,561 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:12:57,562 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:12:57,563 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:12:58,189 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:12:58,449 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:12:58,450 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:12:59,189 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:00,190 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:00,564 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:01,190 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:02,190 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:03,190 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:04,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:05,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:05,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:05,782 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:13:05,782 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:13:05,783 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:13:05,784 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:13:06,264 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:06,457 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:06,457 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:13:08,459 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:11,190 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:13,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:13:13,591 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:13:13,633 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:14,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:14,897 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:13:14,897 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:13:14,898 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:13:14,899 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:13:15,466 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:13:15,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:16,467 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:16,591 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:16,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:21,900 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:23,867 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:24,043 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:13:24,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:25,265 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:13:25,265 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:13:25,266 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:13:25,281 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:13:25,475 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:13:25,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:26,476 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:26,865 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:27,281 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:32,282 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:36,658 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:36,659 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:13:36,659 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:13:37,657 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:37,871 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:38,425 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:13:38,426 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:13:38,426 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:13:38,428 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:13:38,487 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:13:38,657 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:39,657 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:40,488 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:40,657 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:41,658 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:42,657 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:43,658 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:43,775 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:44,775 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:45,121 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:13:45,122 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:13:45,122 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:13:45,151 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:13:45,493 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:13:45,775 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:46,494 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:46,775 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:49,658 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:51,736 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:13:51,736 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:51,736 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:13:52,735 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:53,109 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:13:53,110 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:13:53,110 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:13:53,111 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:13:53,501 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:13:53,735 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:54,043 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:13:54,502 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:13:54,735 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:55,044 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:13:55,735 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:56,736 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:13:59,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:00,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:00,621 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:00,661 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:00,661 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:00,662 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:00,664 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:01,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:01,509 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:02,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:02,510 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:03,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:04,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:05,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:05,736 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:06,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:07,881 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:14:07,881 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:14:07,921 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:08,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:09,597 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:09,597 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:09,598 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:09,599 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:09,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:10,518 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:10,518 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:10,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:11,600 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:11,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:15,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:16,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:16,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:17,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:17,013 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:17,014 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:17,015 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:17,524 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:17,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:18,525 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:18,549 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:22,016 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:24,045 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:14:24,442 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:24,442 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:14:24,442 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:14:25,441 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:25,793 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:25,793 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:25,794 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:25,796 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:26,441 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:26,532 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:26,532 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:27,441 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:27,797 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:28,441 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:29,442 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:33,442 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:34,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:35,666 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:36,044 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:36,044 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:36,044 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:36,046 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:36,541 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:36,666 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:38,542 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:39,046 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:40,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:40,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:14:40,577 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:14:41,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:42,041 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:42,041 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:42,042 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:42,043 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:42,547 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:42,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:43,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:44,548 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:45,044 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:48,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:49,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:50,212 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:50,253 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:50,253 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:50,254 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:50,255 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:50,568 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:50,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:51,647 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:52,569 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:52,648 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:54,046 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:14:56,047 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:14:56,286 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:56,286 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:14:56,286 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:14:57,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:57,593 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:14:57,593 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:14:57,594 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:14:57,595 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:14:58,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:14:58,574 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:14:58,574 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:14:59,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:00,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:01,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:01,286 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:02,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:03,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:04,277 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:04,278 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:04,278 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:04,279 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:04,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:04,580 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:05,285 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:06,286 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:06,582 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:07,286 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:11,813 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:11,814 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:15:11,814 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:15:12,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:13,037 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:13,201 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:13,201 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:13,202 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:13,203 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:13,589 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:13,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:14,590 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:14,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:15,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:18,204 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:18,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:19,379 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:20,005 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:20,005 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:20,006 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:20,007 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:20,379 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:20,596 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:21,379 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:22,379 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:22,598 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:23,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:23,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:24,048 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:15:24,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:25,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:26,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:26,812 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:15:26,813 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:15:27,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:28,533 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:28,535 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:29,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:30,089 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:30,089 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:30,090 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:30,093 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:30,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:30,606 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:31,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:32,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:32,607 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:33,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:34,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:35,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:36,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:36,369 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:36,369 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:36,370 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:36,387 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:36,611 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:37,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:38,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:38,613 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:39,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:40,001 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:40,534 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:42,414 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:15:42,414 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:15:42,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:43,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:43,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:43,714 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:43,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:43,715 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:44,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:44,619 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:44,620 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:45,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:45,716 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:46,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:50,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:51,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:51,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:51,665 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:51,681 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:51,681 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:51,682 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:52,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:52,627 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:15:52,627 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:15:53,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:54,049 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:15:54,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:55,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:56,503 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:57,342 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:15:57,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:15:57,413 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:15:57,503 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:57,905 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:15:57,905 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:15:57,906 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:15:57,907 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:15:58,503 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:15:58,633 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:16:00,635 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:16:02,908 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:05,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:06,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:07,157 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:16:07,157 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:16:07,158 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:16:07,159 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:16:07,641 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:16:07,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:08,159 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:08,642 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:16:08,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:09,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:10,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:11,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:12,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:16:12,413 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:16:12,765 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:13,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:13,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:14,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:15,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:16,553 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:16:16,553 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:16:16,554 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:16:16,555 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:16:16,661 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:16:16,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:17,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:18,663 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:16:18,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:19,555 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:19,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:20,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:24,050 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:16:25,052 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:25,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:26,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:27,305 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:16:27,305 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:16:27,306 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:16:27,307 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:16:27,413 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:16:27,413 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:16:27,651 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:27,672 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:16:28,673 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:16:30,634 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:33,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:34,107 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:34,453 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:16:34,453 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:16:34,454 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:16:34,456 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:16:34,679 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:16:35,107 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:36,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:36,456 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:36,681 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:16:37,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:38,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:39,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:42,108 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:42,896 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:42,896 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:16:42,897 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:16:43,896 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:44,245 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:16:44,245 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:16:44,246 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:16:44,247 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:16:44,689 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:16:44,896 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:46,691 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:16:47,247 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:52,248 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:54,053 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:16:58,054 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:16:58,309 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:58,309 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:16:58,309 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:16:59,308 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:16:59,669 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:16:59,669 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:16:59,670 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:16:59,671 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:16:59,702 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:00,308 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:00,703 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:01,308 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:03,671 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:03,672 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:04,671 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:04,977 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:04,978 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:04,978 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:04,980 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:05,671 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:05,708 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:06,709 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:08,980 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:12,459 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:13,309 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:17:13,309 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:17:13,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:13,873 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:13,873 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:13,874 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:13,875 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:14,458 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:14,716 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:14,876 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:16,721 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:19,876 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:20,023 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:21,020 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:21,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:21,367 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:21,367 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:21,368 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:21,725 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:22,020 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:22,726 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:23,020 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:24,054 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:17:25,055 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:27,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:28,309 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:17:28,309 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:17:28,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:29,353 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:29,353 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:29,354 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:29,355 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:29,734 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:29,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:30,356 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:30,735 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:30,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:31,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:32,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:33,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:34,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:35,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:35,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:36,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:37,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:38,405 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:38,405 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:38,406 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:38,408 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:38,766 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:38,995 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:39,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:40,768 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:40,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:41,409 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:41,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:42,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:43,309 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:17:43,309 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:17:44,698 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:45,698 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:46,009 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:46,009 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:46,010 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:46,025 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:46,698 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:46,789 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:47,025 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:48,791 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:52,026 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:52,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:53,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:53,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:17:53,477 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:17:53,478 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:17:53,479 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:17:53,796 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:17:54,055 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:17:54,143 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:54,797 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:17:55,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:56,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:57,057 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:17:57,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:59,464 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:17:59,465 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:17:59,465 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:18:00,464 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:00,933 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:00,935 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:00,935 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:00,937 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:01,464 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:01,833 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:02,144 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:02,834 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:06,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:07,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:07,145 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:07,457 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:07,457 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:07,457 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:07,459 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:07,839 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:08,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:08,840 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:09,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:10,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:11,105 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:12,460 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:13,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:14,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:14,464 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:18:14,464 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:18:14,727 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:14,728 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:14,728 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:14,731 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:14,846 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:15,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:16,380 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:16,848 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:17,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:17,731 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:18,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:19,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:20,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:21,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:22,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:23,216 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:23,257 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:23,257 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:23,258 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:23,259 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:23,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:23,855 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:24,057 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:18:24,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:24,856 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:25,381 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:29,058 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:29,492 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:29,492 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:18:29,493 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:18:30,491 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:30,977 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:30,977 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:30,978 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:30,979 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:31,491 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:31,863 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:32,864 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:34,980 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:36,099 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:37,098 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:37,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:37,390 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:37,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:37,407 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:37,869 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:38,098 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:38,870 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:40,408 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:43,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:44,491 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:18:44,492 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:18:44,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:44,969 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:44,969 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:44,970 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:44,971 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:45,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:45,876 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:45,972 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:46,877 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:50,972 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:53,083 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:54,058 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:18:54,082 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:54,465 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:18:54,465 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:18:54,466 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:18:54,467 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:18:54,885 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:18:55,082 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:56,083 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:56,467 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:18:56,896 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:18:57,083 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:58,083 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:59,083 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:18:59,491 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:18:59,491 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:19:01,684 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:01,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:02,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:03,345 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:19:03,346 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:19:03,346 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:19:03,347 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:19:03,903 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:19:03,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:04,904 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:19:04,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:07,348 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:12,348 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:17,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:19:17,313 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:19:17,353 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:17,572 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:18,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:18,677 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:19:18,677 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:19:18,678 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:19:18,679 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:19:18,916 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:19:19,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:20,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:20,918 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:19:21,312 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:22,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:24,061 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:19:25,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:26,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:26,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:19:26,725 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:19:26,726 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:19:26,727 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:19:26,924 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:19:27,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:27,727 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:28,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:28,925 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:19:29,329 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:32,728 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:33,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:33,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:19:33,701 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:19:34,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:35,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:19:35,193 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:19:35,194 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:19:35,225 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:19:35,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:35,948 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:19:36,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:36,948 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:19:37,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:38,225 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:38,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:39,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:40,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:41,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:42,197 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:19:42,197 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:19:42,198 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:19:42,199 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:19:42,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:42,955 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:19:43,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:43,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:44,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:44,956 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:19:45,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:46,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:47,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:48,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:19:48,700 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:19:48,741 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:48,946 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:49,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:50,565 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:19:50,565 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:19:50,566 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:19:50,568 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:19:50,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:50,963 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:19:51,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:52,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:52,964 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:19:53,701 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:19:54,061 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:19:54,062 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:59,062 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:19:59,523 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:00,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:00,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:00,989 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:00,990 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:00,991 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:01,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:01,973 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:02,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:02,974 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:03,522 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:03,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:20:03,700 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:20:04,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:04,711 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:05,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:05,996 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:05,996 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:05,996 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:05,998 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:06,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:06,978 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:07,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:08,710 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:08,980 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:09,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:14,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:15,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:16,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:16,461 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:16,461 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:16,462 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:16,463 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:16,987 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:17,044 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:18,044 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:18,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:20:18,700 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:20:18,989 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:19,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:20,004 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:20,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:22,566 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:23,566 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:23,985 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:23,986 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:23,986 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:23,988 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:23,994 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:24,063 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:20:24,567 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:25,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:25,567 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:26,997 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:30,046 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:31,610 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:32,609 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:32,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:32,961 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:32,962 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:32,963 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:33,002 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:33,609 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:33,700 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:20:33,700 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:20:34,609 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:35,004 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:35,609 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:35,998 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:36,610 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:40,047 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:41,047 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:41,385 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:41,425 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:41,425 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:41,426 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:41,427 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:42,011 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:42,047 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:43,012 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:43,047 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:44,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:45,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:46,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:46,609 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:47,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:51,610 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:53,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:53,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:20:53,102 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:20:54,063 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:20:54,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:54,509 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:20:54,509 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:20:54,510 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:20:54,511 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:20:55,040 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:20:55,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:56,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:57,042 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:20:57,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:57,512 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:20:58,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:20:59,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:00,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:01,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:01,821 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:01,822 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:01,822 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:01,823 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:02,047 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:02,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:02,823 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:03,048 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:03,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:04,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:05,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:06,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:07,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:08,102 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:08,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:08,703 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:21:08,703 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:21:09,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:10,029 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:10,029 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:10,030 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:10,031 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:10,055 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:10,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:13,058 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:14,032 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:15,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:16,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:16,573 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:16,574 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:16,574 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:16,575 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:17,077 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:17,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:18,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:19,078 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:19,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:19,576 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:20,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:21,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:22,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:23,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:23,702 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:21:23,702 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:21:24,065 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:21:24,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:24,889 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:24,929 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:24,931 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:24,931 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:24,932 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:25,084 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:25,164 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:26,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:27,092 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:27,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:28,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:29,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:30,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:30,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:31,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:32,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:32,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:32,887 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:32,887 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:32,889 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:33,098 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:33,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:34,165 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:35,100 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:35,889 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:38,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:38,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:21:38,894 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:21:39,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:40,305 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:40,305 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:40,306 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:40,307 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:40,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:41,106 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:41,308 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:41,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:43,108 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:46,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:47,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:47,244 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:47,593 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:47,593 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:47,594 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:47,595 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:48,113 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:48,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:49,114 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:49,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:50,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:51,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:53,222 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:53,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:53,893 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:21:53,893 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:21:54,106 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:21:54,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:54,993 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:21:54,993 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:21:54,994 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:21:54,995 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:21:55,119 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:21:55,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:56,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:57,135 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:21:57,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:58,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:21:59,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:21:59,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:00,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:01,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:02,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:03,045 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:03,046 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:03,046 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:03,047 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:03,141 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:03,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:04,590 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:05,048 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:05,143 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:09,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:09,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:22:09,590 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:22:10,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:10,802 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:10,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:10,990 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:10,990 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:10,991 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:11,149 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:11,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:12,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:13,151 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:13,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:14,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:15,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:16,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:16,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:17,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:18,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:18,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:18,738 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:18,738 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:18,739 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:19,157 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:19,688 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:20,688 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:21,159 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:21,688 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:22,589 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:22,688 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:24,067 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:22:27,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:27,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:22:27,284 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:22:28,283 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:28,487 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:28,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:28,709 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:28,710 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:28,711 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:29,166 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:29,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:30,284 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:31,168 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:33,712 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:34,926 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:35,926 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:36,373 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:36,373 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:36,374 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:36,375 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:36,926 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:37,174 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:37,926 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:38,926 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:39,176 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:39,376 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:39,927 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:43,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:43,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:22:43,016 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:22:44,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:44,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:44,390 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:44,390 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:44,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:44,391 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:45,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:45,196 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:46,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:47,212 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:49,927 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:50,478 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:51,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:51,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:51,858 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:51,858 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:51,860 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:52,217 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:52,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:53,218 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:53,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:54,069 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:22:54,219 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:22:54,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:55,070 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:22:55,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:56,731 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:57,731 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:58,113 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:22:58,113 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:22:58,114 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:22:58,338 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:22:58,338 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:22:58,340 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:22:58,731 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:22:59,224 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:22:59,732 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:00,341 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:00,732 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:01,226 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:04,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:05,477 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:05,601 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:05,965 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:05,965 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:05,966 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:05,967 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:06,245 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:09,247 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:10,968 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:11,840 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:12,839 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:13,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:23:13,221 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:13,222 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:23:13,527 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:13,527 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:13,529 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:13,839 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:14,253 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:14,839 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:15,267 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:15,839 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:16,530 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:16,840 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:19,145 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:20,146 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:20,521 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:20,521 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:20,522 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:20,523 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:21,146 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:21,273 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:21,840 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:23,275 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:24,070 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:23:27,071 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:27,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:28,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:23:28,177 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:23:28,680 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:28,989 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:28,989 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:28,990 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:28,991 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:29,281 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:29,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:30,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:31,283 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:31,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:32,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:32,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:33,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:34,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:35,469 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:35,469 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:35,470 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:35,471 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:35,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:36,289 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:36,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:37,289 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:37,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:37,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:38,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:42,681 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:42,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:43,177 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:23:43,177 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:23:43,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:44,193 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:44,193 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:44,194 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:44,208 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:44,296 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:44,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:45,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:46,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:47,299 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:47,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:47,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:48,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:49,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:51,377 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:52,377 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:52,813 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:23:52,814 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:23:52,814 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:23:52,815 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:23:53,304 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:23:53,377 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:53,822 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:54,073 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:23:54,377 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:55,306 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:23:58,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:23:58,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:23:58,805 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:23:59,039 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:23:59,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:00,513 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:00,514 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:00,514 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:00,515 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:00,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:01,312 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:01,804 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:02,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:03,314 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:04,516 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:08,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:09,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:09,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:10,293 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:10,294 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:10,294 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:10,295 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:10,331 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:10,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:11,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:12,333 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:12,962 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:13,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:24:13,805 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:24:13,961 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:15,100 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:18,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:19,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:19,709 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:19,709 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:19,710 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:19,724 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:20,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:20,340 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:20,725 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:21,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:21,341 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:24,072 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:24:26,073 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:26,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:27,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:27,689 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:27,690 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:27,690 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:27,691 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:28,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:28,348 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:28,805 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:24:28,805 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:24:29,313 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:29,349 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:31,999 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:33,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:34,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:34,713 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:34,714 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:34,714 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:34,716 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:35,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:35,368 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:36,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:37,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:37,370 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:37,717 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:38,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:39,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:40,367 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:41,367 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:43,367 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:47,856 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:47,857 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:24:47,857 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:24:48,856 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:49,128 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:49,185 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:49,185 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:49,186 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:49,187 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:49,381 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:49,856 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:51,383 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:54,075 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:24:55,076 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:24:55,756 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:56,755 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:57,389 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:24:57,389 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:24:57,390 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:24:57,391 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:24:57,755 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:58,389 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:24:58,755 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:24:59,390 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:24:59,755 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:00,391 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:00,755 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:01,756 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:03,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:25:03,307 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:25:03,349 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:04,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:04,737 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:25:04,738 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:25:04,738 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:25:04,740 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:25:05,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:05,397 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:25:05,740 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:06,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:07,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:07,412 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:08,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:09,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:10,307 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:10,756 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:11,767 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:12,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:13,169 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:25:13,170 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:25:13,170 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:25:13,171 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:25:13,417 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:25:13,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:14,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:15,435 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:15,766 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:16,171 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:16,767 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:19,128 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:19,129 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:25:19,129 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:25:20,128 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:20,545 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:25:20,545 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:25:20,546 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:25:20,547 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:25:21,129 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:21,441 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:25:21,547 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:22,129 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:22,441 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:23,129 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:24,075 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:25:27,076 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:27,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:28,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:29,101 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:25:29,101 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:25:29,102 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:25:29,103 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:25:29,463 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:25:29,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:30,464 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:30,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:31,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:32,104 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:32,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:33,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:34,128 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:25:34,129 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:25:34,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:35,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:36,524 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:37,353 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:37,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:38,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:38,884 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:25:38,884 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:25:38,884 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:25:38,886 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:25:39,473 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:25:39,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:40,474 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:40,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:41,525 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:42,886 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:44,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:45,879 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:46,569 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: partial_history +2024-09-21 12:25:46,570 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:25:46,570 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:25:46,571 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:25:47,029 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:47,480 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:25:48,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:48,030 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:48,481 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:49,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:49,128 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:25:49,129 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:25:50,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:51,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:52,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:53,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:53,879 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:25:54,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:54,077 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:25:55,013 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:56,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:56,487 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:25:57,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:58,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:59,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:25:59,226 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:00,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:01,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:02,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:03,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:04,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:04,129 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:26:04,129 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:26:04,379 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:05,014 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:06,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:07,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:08,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:09,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:09,728 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:10,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:10,515 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:26:11,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:12,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:13,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:14,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:14,879 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:15,015 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:17,971 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:18,521 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:26:18,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:19,129 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: stop_status +2024-09-21 12:26:19,129 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: stop_status +2024-09-21 12:26:19,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:19,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:20,959 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:21,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:22,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:23,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:24,078 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:26:24,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:24,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:25,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:26,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:27,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:28,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:29,880 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:29,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:30,960 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:31,233 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: server_info +2024-09-21 12:26:31,233 DEBUG SenderThread:2812040 [sender.py:send():379] send: telemetry +2024-09-21 12:26:31,233 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: server_info +2024-09-21 12:26:31,422 DEBUG SenderThread:2812040 [sender.py:send():379] send: exit +2024-09-21 12:26:31,422 INFO SenderThread:2812040 [sender.py:send_exit():586] handling exit code: 0 +2024-09-21 12:26:31,422 INFO SenderThread:2812040 [sender.py:send_exit():588] handling runtime: 4447 +2024-09-21 12:26:31,424 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:26:31,424 INFO SenderThread:2812040 [sender.py:send_exit():594] send defer +2024-09-21 12:26:31,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:31,424 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 0 +2024-09-21 12:26:31,424 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:31,424 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-09-21 12:26:31,424 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 1 +2024-09-21 12:26:31,424 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:31,424 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 1 +2024-09-21 12:26:31,425 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:31,425 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-09-21 12:26:31,425 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 2 +2024-09-21 12:26:31,425 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:31,425 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 2 +2024-09-21 12:26:31,425 INFO HandlerThread:2812040 [system_monitor.py:finish():203] Stopping system monitor +2024-09-21 12:26:31,425 DEBUG SystemMonitor:2812040 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-09-21 12:26:31,425 DEBUG SystemMonitor:2812040 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-09-21 12:26:31,425 INFO HandlerThread:2812040 [interfaces.py:finish():200] Joined cpu monitor +2024-09-21 12:26:31,426 INFO HandlerThread:2812040 [interfaces.py:finish():200] Joined disk monitor +2024-09-21 12:26:31,532 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:26:32,533 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:26:32,619 INFO HandlerThread:2812040 [interfaces.py:finish():200] Joined gpu monitor +2024-09-21 12:26:32,619 INFO HandlerThread:2812040 [interfaces.py:finish():200] Joined memory monitor +2024-09-21 12:26:32,619 INFO HandlerThread:2812040 [interfaces.py:finish():200] Joined network monitor +2024-09-21 12:26:32,619 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-21 12:26:32,619 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:32,619 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-09-21 12:26:32,619 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 3 +2024-09-21 12:26:32,620 DEBUG SenderThread:2812040 [sender.py:send():379] send: stats +2024-09-21 12:26:32,620 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: poll_exit +2024-09-21 12:26:32,620 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:32,620 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 3 +2024-09-21 12:26:32,621 DEBUG SenderThread:2812040 [sender.py:send():379] send: history +2024-09-21 12:26:32,621 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: summary_record +2024-09-21 12:26:32,622 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:26:32,622 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:32,623 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-09-21 12:26:32,623 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 4 +2024-09-21 12:26:32,623 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:32,623 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 4 +2024-09-21 12:26:32,623 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:32,623 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-09-21 12:26:32,623 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 5 +2024-09-21 12:26:32,623 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:32,623 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 5 +2024-09-21 12:26:32,623 DEBUG SenderThread:2812040 [sender.py:send():379] send: summary +2024-09-21 12:26:32,624 INFO SenderThread:2812040 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-09-21 12:26:32,624 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:32,624 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-09-21 12:26:32,624 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 6 +2024-09-21 12:26:32,625 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:32,625 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 6 +2024-09-21 12:26:32,625 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:32,625 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-09-21 12:26:32,626 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: status_report +2024-09-21 12:26:32,826 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 7 +2024-09-21 12:26:32,826 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:32,826 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 7 +2024-09-21 12:26:32,826 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:32,826 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-09-21 12:26:33,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-21 12:26:33,535 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/config.yaml +2024-09-21 12:26:33,535 INFO Thread-12 :2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:26:34,439 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 8 +2024-09-21 12:26:34,439 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: poll_exit +2024-09-21 12:26:34,439 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:34,439 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 8 +2024-09-21 12:26:34,439 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:34,439 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-09-21 12:26:34,440 INFO SenderThread:2812040 [job_builder.py:build():440] Attempting to build job artifact +2024-09-21 12:26:34,440 INFO SenderThread:2812040 [job_builder.py:_get_source_type():569] is repo sourced job +2024-09-21 12:26:34,440 WARNING SenderThread:2812040 [job_builder.py:_log_if_verbose():274] No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job +2024-09-21 12:26:34,440 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 9 +2024-09-21 12:26:34,441 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:34,441 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 9 +2024-09-21 12:26:34,441 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:34,441 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-09-21 12:26:34,441 INFO SenderThread:2812040 [dir_watcher.py:finish():358] shutting down directory watcher +2024-09-21 12:26:34,536 INFO SenderThread:2812040 [dir_watcher.py:_on_file_modified():288] file/dir modified: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:26:34,536 INFO SenderThread:2812040 [dir_watcher.py:finish():388] scan: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files +2024-09-21 12:26:34,536 INFO SenderThread:2812040 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/conda-environment.yaml conda-environment.yaml +2024-09-21 12:26:34,537 INFO SenderThread:2812040 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log output.log +2024-09-21 12:26:34,537 INFO SenderThread:2812040 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-metadata.json wandb-metadata.json +2024-09-21 12:26:34,537 INFO SenderThread:2812040 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/requirements.txt requirements.txt +2024-09-21 12:26:34,537 INFO SenderThread:2812040 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json wandb-summary.json +2024-09-21 12:26:34,537 INFO SenderThread:2812040 [dir_watcher.py:finish():402] scan save: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/config.yaml config.yaml +2024-09-21 12:26:34,537 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 10 +2024-09-21 12:26:34,537 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:34,539 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 10 +2024-09-21 12:26:34,564 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:34,565 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-09-21 12:26:34,566 INFO SenderThread:2812040 [file_pusher.py:finish():169] shutting down file pusher +2024-09-21 12:26:35,093 INFO wandb-upload_4:2812040 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/config.yaml +2024-09-21 12:26:35,104 INFO wandb-upload_2:2812040 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/requirements.txt +2024-09-21 12:26:35,140 INFO wandb-upload_0:2812040 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/conda-environment.yaml +2024-09-21 12:26:35,143 INFO wandb-upload_3:2812040 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/wandb-summary.json +2024-09-21 12:26:35,165 INFO wandb-upload_1:2812040 [upload_job.py:push():130] Uploaded file ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/files/output.log +2024-09-21 12:26:35,365 INFO Thread-11 (_thread_body):2812040 [sender.py:transition_state():614] send defer: 11 +2024-09-21 12:26:35,366 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:35,366 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 11 +2024-09-21 12:26:35,366 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:35,366 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-09-21 12:26:35,366 INFO SenderThread:2812040 [file_pusher.py:join():175] waiting for file pusher +2024-09-21 12:26:35,367 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 12 +2024-09-21 12:26:35,367 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:35,367 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 12 +2024-09-21 12:26:35,367 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:35,367 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-09-21 12:26:35,367 INFO SenderThread:2812040 [file_stream.py:finish():601] file stream finish called +2024-09-21 12:26:35,422 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-21 12:26:36,112 INFO SenderThread:2812040 [file_stream.py:finish():605] file stream finish is done +2024-09-21 12:26:36,112 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 13 +2024-09-21 12:26:36,112 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: poll_exit +2024-09-21 12:26:36,112 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:36,112 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 13 +2024-09-21 12:26:36,113 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:36,113 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-09-21 12:26:36,113 INFO SenderThread:2812040 [sender.py:transition_state():614] send defer: 14 +2024-09-21 12:26:36,113 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: defer +2024-09-21 12:26:36,113 DEBUG SenderThread:2812040 [sender.py:send():379] send: final +2024-09-21 12:26:36,113 INFO HandlerThread:2812040 [handler.py:handle_request_defer():184] handle defer: 14 +2024-09-21 12:26:36,113 DEBUG SenderThread:2812040 [sender.py:send():379] send: footer +2024-09-21 12:26:36,113 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: defer +2024-09-21 12:26:36,113 INFO SenderThread:2812040 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-09-21 12:26:36,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: poll_exit +2024-09-21 12:26:36,114 DEBUG SenderThread:2812040 [sender.py:send_request():406] send_request: poll_exit +2024-09-21 12:26:36,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: internal_messages +2024-09-21 12:26:36,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: get_summary +2024-09-21 12:26:36,114 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: sampled_history +2024-09-21 12:26:36,115 DEBUG HandlerThread:2812040 [handler.py:handle_request():158] handle_request: shutdown +2024-09-21 12:26:36,115 INFO HandlerThread:2812040 [handler.py:finish():882] shutting down handler +2024-09-21 12:26:37,114 INFO WriterThread:2812040 [datastore.py:close():296] close: ../outputs/dpo_tiv2t_1.5k_base/wandb/run-20240921_111222-4ony16ec/run-4ony16ec.wandb +2024-09-21 12:26:37,114 INFO SenderThread:2812040 [sender.py:finish():1615] shutting down sender +2024-09-21 12:26:37,114 INFO SenderThread:2812040 [file_pusher.py:finish():169] shutting down file pusher +2024-09-21 12:26:37,114 INFO SenderThread:2812040 [file_pusher.py:join():175] waiting for file pusher